Java实现十秒向MySQL插入百万条数据
目录
- mysql数据库准备
- 方式一：普通插入
- 方式二：使用批处理插入
- 方式三：通过连接配置url设置【&rewriteBATchedStatements=true】（设置重写批处理语句）
- 方式四：通过数据库连接取消自动提交，手动提交数据
mysql数据库准备
private String Driver = "com.mysql.cj.jdbc.Driver";
private String url ="jdbc:mysql://localhost:3306/mp?serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true";
private String user = "root";
private String password = "root";
Connection connection = null;
PreparedStatement ps = null;
ResultSet rs = null;
//封装与数据库建立连接的类
public void coon() throws Exception{
Class.forName(Driver);
connection = DriverManager.getConnection(url,user,password);
}
//封装异常类
public void erro(){
try {
if (rs!=null){
rs.close();
}
if (ps!=null){
ps.close();
}
if (connection!=null){
connection.close();
}
} catch (Exception e) {
e.printStackTraceLKEXg();
}
}
方式一：普通插入
package com.wt;
import org.junit.Test;
import Java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
/**
* @Author wt
* @Date 2022/11/14 21:17
* @PackageName:com.wt
* @ClassName: TestAddBatch01
* @Description: TODO
* @Version 1.0
*/
public class TestAddBatch01 {
private String Driver = "com.mysql.cj.jdbc.Driver";
private String url ="jdbc:mysql://localhost:3306/mp?serverTimezone=Asia/Shanghai";
private String user = "root";
private String password = "root";
Connection connection = null;
PreparedStatement ps = null;
ResultSet rs = null;
public void coon() throws Exception{
Class.forName(Driver);
connection = DriverManager.getConnection(url,user,password);
}
public void erro(){
try {
if (rs!=null){
rs.close();
}
if (ps!=null){
ps.clos开发者_JAVA开发e();
}
if (connection!=null){
connection.close();
}
} catch (Exception e) {
e.printStackTrace();
}
}
@Test
public void ccc(){
long start = System.currentTimeMillis();
String sql = "insert into a(id, name) VALUES (?,null)";
try {
coon();
ps = connection.prepareStatement(sql);
for (int i = 1; i <= 1000000; i++) {
ps.setObject(1, i);//填充sql语句种得占位符
ps.execute();//执行sql语句
}
} catch (Exception e) {
e.printStackTrace();
} finally {
erro();
}
System.out.println("百万条数据插入用时：" + (System.currentTimeMillis() - start)+"【单位：毫秒】");
}
}
用时：62分钟多
方式二：使用批处理插入
package com.wt;
import org.junit.Test;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
/**
* @Author wt
* @Date 2022/11/14 20:25
* @PackageName:com.wt.util
* @ClassName: TestAddBatch
* @Description: TODO
* @Version 1.0
*/
public class TestAddBatch {
private String Driver = "com.mysql.cj.jdbc.Driver";
private String url ="jdbc:mysql://localhost:3306/mp?serverTimezone=Asia/Shanghai";
private String user = "root";
private String password = "root";
Connection connection = null;
PreparedStatement ps = null;
ResultSet rs = null;
public void coon() throws Exception{
Class.forName(Driver);
connection = DriverManager.getConnection(url,user,password);
}
public void erro(){
try {
if (rs!=null){
rs.close();
}
if (ps!=null){
ps.close();
}
if (connection!=null){
connection.close();
}
} catch (Exception e) {
e.printStackTrace();
}
}
@Test
public void ccc(){
long start = System.currentTimeMillis();
String sql = "insert into a(id, name) VALUES (?,null)";
try {
coon();
ps = connection.prepareStatement(sql);
// connection.setAutoCommit(false);//取消自动提交
for (int i = 1; i <= 1000000; i++) {
ps.setObject(1, i);
ps.addBatch();
if (i % 1000 == 0) {
ps.executeBatch();
ps.clearBatch();
}
}
ps.executeBatch();
ps.clearBatch();
// connection.commit();//所有语句都执行完毕后才手动提交sql语句
} catch (Exception e) {
e.printStackTrace();
} finally {
erro();
}
System.out.println("百万条数据插入用时：" + (System.currentTimeMillis() - start)+"【单位：毫秒】");
}
}
方式一、二总结：到此可以看出其实其处理程序及批处理是没有起作用的，为此我们使用方式三
方式三：通过连接配置url设置【&rewriteBatchedStatements=true】（设置重写批处理语句）
url地址后注意添加【&rewriteBatchedStatements=true】
private String url ="jdbc:mysql://localhost:3306/mp?serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true";
方法三较于方法二的改变是只是url地址上的改变，其它没有任何修改
package com.wt;
import org.junit.Test;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
/**
* @Author wt
* @Date 2022/11/14 20:25
* @PackageName:com.wt.util
* @ClassName: TestAddBatch
* @Description: TODO
* @Version 1.0
*/
public class TestAddBatch {
private String Driver = "com.mysql.cj.jdbc.Driver";
private String url ="jdbc:mysql://localhost:3306/mp?serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true";
private String user = "root";
private String password = "root";
Connection connection = null;
PreparedStatement ps = null;
ResuandroidltSet rs = null;
public void coon() throws Exception{
Class.forName(Driver);
connection = DriverManager.getConnection(url,user,password);
}
public void erro(){
try {
if (rs!=null){
rs.close();
}
if (ps!=null){
ps.close();
}
if (connection!=null){
connection.close();
}
} catch (Exception e) {
e.printStackTrace();
}
}
@Test
public void ccc(){
long start = System.currentTimeMillis();
String sql = "insert into a(id, name) VALUES (?,null)";
try {
coon();
ps = connection.prepareStatement(sql);
for (int i = 1; i <= 1000000; i++) {
ps.setObject(1, i);
ps.addBatch();
if (i % 1000 == 0) {
ps.executeBatch();
ps.clearBatch();
}
}
ps.executeBatch();
javascript ps.clearBatch();
} catch (Exception e) {
e.printStackTrace();
} finally {
erro();
}
System.out.println("百万条数据插入用时：" + (System.currentTimeMillis() - start)+"【单位：毫秒】");
}
}
用时：【10秒左右】
到此批处理语句才正是生效
注意
数据库连接的url设置了【&rewriteBatchedStatements=true】时，java代码种的sql语句不能有分号【;】号，否则批处理语句打包就会出现错误，导致后面的sql语句提交出现【BatchUpdateException】异常
方式四：通过数据库连接取消自动提交，手动提交数据
package com.wt;
import org.junit.Test;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
/**
* @Author wt
* @Date 2022/11/14 20:25
* @PackageName:com.wt.util
* @ClassName: TestAddBatch
* @Description: TODO
* @Version 1.0
*/
public class TestAddBatch {
private String Driver = "com.mysql.cj.jdbc.Driver";
private String url ="jdbc:mysql://localhost:3306/mp?serverTimezone=Asia/Shanghai&rewriteBatchedStatements=true";
private String user = "root";
private String password = "root";
Connection connection = null;
PreparedStatement ps = null;
ResultSet rs = null;
public void coon() throws Exception{
Class.forName(Driver);
connection = DriverManager.gethttp://www.devze.comConnection(url,user,password);
}
public void erro(){
try {
if (rs!=nul编程客栈l){
rs.close();
}
if (ps!=null){
ps.close();
}
if (connection!=null){
connection.close();
}
} catch (Exception e) {
e.printStackTrace();
}
}
@Test
public void ccc(){
long start = System.currentTimeMillis();
String sql = "insert into a(id, name) VALUES (?,null)";
try {
coon();
ps = connection.prepareStatement(sql);
connection.setAutoCommit(false);//取消自动提交
for (int i = 1; i <= 1000000; i++) {
ps.setObject(1, i);
ps.addBatch();
if (i % 1000 == 0) {
ps.executeBatch();
ps.clearBatch();
}
}
ps.executeBatch();
ps.clearBatch();
connection.commit();//所有语句都执行完毕后才手动提交sql语句
} catch (Exception e) {
e.printStackTrace();
} finally {
erro();
}
System.out.println("百万条数据插入用时：" + (System.currentTimeMillis() - start)+"【单位：毫秒】");
}
}
用时：【9秒左右】
总结：
1.使用批量提交数据，url一定要设置允许重写批量提交【rewriteBatchedStatements=true】，以及此时的sql语句一定不能有分号，否则有【BatchUpdateException】异常，
2.其他的就正常使用PreparedStatement ps;的以下三个方法即可
- ps.addBatch(); 将sql语句打包到一个容器中
- ps.executeBatch(); 将容器中的sql语句提交
- ps.clearBatch(); 清空容器，为下一次打包做准备
以上就是Java实现十秒向MySQL插入百万条数据的详细内容，更多关于Java MySQL插入数据的资料请关注我们其它相关文章！
加载中,请稍侯......
精彩评论