超过100万行数据导出execl,springboot-kotlin项目

x33g5p2x  于2022-06-29 转载在 Spring  
字(4.5k)|赞(0)|评价(0)|浏览(414)

项目中的用量详情,一个月的数据已经超过100万行,需要能够导出来,普通的方法不进导出比较慢,而且会出现OOM问题。

介绍一款可以导出100万行的方法:
https://github.com/alibaba/easyexcel

1、第一种方法:写入10万行,采用分而治之的思想处理

@RequestMapping("/xxxexportcsv100")
	fun export(response: HttpServletResponse) {
		val outputStream: OutputStream = response.outputStream
		val time = SimpleDateFormat("yyyy-MM-dd-hh-mm-ss").format(Date())
		response.setHeader("Content-disposition", "attachment; filename=contract$time.xlsx")
		response.contentType = "application/octet-stream;charset=UTF-8"
		response.setHeader("Pragma", "No-cache")
		response.setHeader("Cache-Control", "no-cache")
		response.setDateHeader("Expires", 0)
		val count: Int = billingDetailService.countByIdIsNotNull()

		val totalStopWatch = StopWatch()
		totalStopWatch.start()
		if (count > 100000) {
			val max = 100000
			val excelWriter: ExcelWriter = EasyExcel.write(outputStream).build()
			for (i in 0 until count / max + 1) {
				val stopWatch = StopWatch()
				stopWatch.start()
				val exportList: List<BillingDetailResourceDto> = billingDetailService.findAllBySize(i, max)
				stopWatch.stop()
				logger.info("查询一次100000行的sql,花费的时间" + stopWatch.totalTimeSeconds)
				val writerStopWatch = StopWatch()
				writerStopWatch.start()
				val writeSheet: WriteSheet =
					EasyExcel.writerSheet(i, "账单" + (i + 1)).head(BillingDetailResourceDto::class.java)
						.registerWriteHandler(LongestMatchColumnWidthStyleStrategy()).build()
				excelWriter.write(exportList, writeSheet)
				writerStopWatch.stop()
				logger.info("执行一次100000行写出到execl,花费的时间${writerStopWatch.totalTimeSeconds}")
			}
			//刷新流
			excelWriter.finish()
		} else {
			val exportList: List<BillingDetailResourceDto> = billingDetailService.findAllBySize(0, count)
			EasyExcel.write(outputStream, BillingDetailResourceDto::class.java)
				.registerWriteHandler(LongestMatchColumnWidthStyleStrategy()).sheet("账单").doWrite(exportList)
		}
		outputStream.flush()
		totalStopWatch.stop()
		logger.info("导出execl总耗时=${totalStopWatch.totalTimeSeconds}")
		response.outputStream.close()

	}

导出的execl列:

测试结果如下:
475276行数据,平局耗时38秒左右

测试结果如下:
1261138行数据,平局耗时113秒左右

第二种方法:每个sheet,写入10万行,每个sheet再次分多次写入

@RequestMapping("/yyyexportcsv100")
	fun writeExcelByMulSheetAndWriteChange(response: HttpServletResponse) {
		val outputStream: OutputStream = response.outputStream
		val time = SimpleDateFormat("yyyy-MM-dd-hh-mm-ss").format(Date())
		response.setHeader("Content-disposition", "attachment; filename=contract$time.xlsx")
		response.contentType = "application/octet-stream;charset=UTF-8"
		response.setHeader("Pragma", "No-cache")
		response.setHeader("Cache-Control", "no-cache")
		response.setDateHeader("Expires", 0)

		//每个sheet写入的数据
		val NUM_PER_SHEET = 100000
		//每次向sheet中写入的数据
		val NUM_BY_TIMES = 50000
		val startTime = System.currentTimeMillis()
		// 获取数据
		val count: Int = billingDetailService.countByIdIsNotNull()
		// 获取 sheet 的个数
		val sheetNum: Int =
			if (count % NUM_PER_SHEET === 0) count / NUM_PER_SHEET else count / NUM_PER_SHEET + 1
		// 获取每个sheet 写入的次数
		val writeNumPerSheet: Int =
			if (NUM_PER_SHEET % NUM_BY_TIMES === 0) NUM_PER_SHEET / NUM_BY_TIMES else NUM_PER_SHEET / NUM_BY_TIMES + 1
		// 最后一个 sheet 写入的数量
		val writeNumLastSheet: Int = count - (sheetNum - 1) * NUM_PER_SHEET
		// 最后一个 sheet 写入的次数
		val writeNumPerLastSheet: Int =
			if (writeNumLastSheet % NUM_BY_TIMES === 0) writeNumLastSheet / NUM_BY_TIMES else writeNumLastSheet / NUM_BY_TIMES + 1
		// 指定写入的文件
		val excelWriter = EasyExcel.write(outputStream, BillingDetailResourceDto::class.java).build()
		for (i in 0 until sheetNum) {
			val sheetName = "sheet$i"
			val writeSheet = EasyExcel.writerSheet(i, sheetName).build()
			val writeNum = if (i == sheetNum - 1) writeNumPerLastSheet else writeNumPerSheet // 每个sheet 写入的次数
			val endEndNum = if (i == sheetNum - 1) count else (i + 1) * NUM_PER_SHEET // 每个sheet 最后一次写入的最后行数
			for (j in 0 until writeNum) {
				val l = System.currentTimeMillis()
				val startNum: Int = i * NUM_PER_SHEET + j * NUM_BY_TIMES
				val endNum = if (j == writeNum - 1) endEndNum else i * NUM_PER_SHEET + (j + 1) * NUM_BY_TIMES
				val stopWatch = StopWatch()
				stopWatch.start()
				val exportList: List<BillingDetailResourceDto> = billingDetailService.findAllBySize(i, endNum - startNum)
				stopWatch.stop()
				logger.info("查询一次100000行的sql,花费的时间" + stopWatch.totalTimeSeconds)
				excelWriter!!.write(exportList, writeSheet)
				logger.info(
					"写入sheet={},数据量={}-{}={},耗时={}",
					sheetName,
					endNum,
					startNum,
					endNum - startNum,
					System.currentTimeMillis() - l
				)
			}
		}
		// 需要放入 finally 中
		excelWriter?.finish()
		logger.info("导出excel结束,总数据量={},耗时={}ms", count, System.currentTimeMillis() - startTime)
		response.outputStream.close()
	}

测试结果如下:
475276行数据,平局耗时37秒左右

测试结果如下:
1261138行数据,平局耗时106秒左右

两种方法相差不多,100万行的数据导出,还是需要从产品角度考虑,不能让用户一直在页面等待。

相关文章