水印总是Long.MIN_VALUE问题

在Flink 1.7版本中,使用FlinkKafkaConsumer010进行事件时间处理时遇到了水印总是为Long.MIN_VALUE的问题,导致水位线无法触发窗口。问题源于缺省的FixedPartitioner导致某些channelstatus的水位线保持在最小值,从而影响整体水位线的输出。尽管尝试调整TimestampAndWatermarkAssigner的并行度,但问题未得到解决。解决方案在于理解FlinkKafkaConsumer的分区行为,并关注空闲分区的水位线处理,该问题在flink1.11版本通过添加withIdleness选项得到了修复。

项目场景:

flink1.7版本:事件时间处理时,水印总是Long.MIN_VALUE问题

问题描述

水位线怎么不触发?数据一直有序得进来,为什么没有窗口被fire掉

原因分析:

Flink source用到了FlinkKafkaConsumer010,没有指定KafkaPartitioner的话,会通过FixedPartitioner来给出默认的partitioner方法:
public int partition(T record, byte[] key, byte[] value, String targetTopic, int[] partitions) {
Preconditions.checkArgument(partitions != null && partitions.length > 0, “Partitions of the target topic is empty.”);
return partitions[this.parallelInstanceId % partitions.length];
}

追踪源码:
StatusWatermarkValve.java
private void findAndOutputNewMinWatermarkAcrossAlignedChannels() {
long newMinWatermark = Long.MAX_VALUE;
// determine new overall watermark by considering only watermark-aligned channels across all channels
for (InputChannelStatus channelStatus : channelStatuses) {
if (channelStatus.isWatermarkAligned) {
newMinWatermark = Math.min(channelStatus.watermark, newMinWatermark);
}
}
// we acknowledge and output the new overall watermark if it is larger than the last output watermark
if (newMinWatermark > lastOutputWatermark) {
lastOutputWatermark = newMinWatermark;
outputHandler.handleWatermark(new Watermark(lastOutputWatermark));
}
}
这里会将所有的channel status的水位线做个汇总:取最小的水位线。这个地方有的channel status下的水位线一直是最小的long值那个不正常的水位线,进而导致整体的水位线发送不出去

解决方案:

网上解决方案:修改assigntimestampandwatermarks的并行度
当assigntimestampandwatermarks的并行度修改后,事件会被shuffled across(洗牌),因此到了TimestampAndWatermarkAssigner不会有空的partition存在了。
并未解决我遇到的问题,根据FlinkKafkaConsumer 中的 per-partition watermarks 应该考虑空闲分区的讨论,解决方案在flink1.11被修复,增加了withIdleness

public ByteArrayOutputStream exportToWordStream(List<EntityItem> records) throws Exception { // 创建临时文件作为缓冲区 File tempFile = File.createTempFile("word_export", ".docx"); try (FileOutputStream fos = new FileOutputStream(tempFile); XWPFDocument doc = new XWPFDocument()) { // 设置页面布局 setupPageLayout(doc); // 初始化表格和计数器 XWPFTable currentTable = createNewTable(doc); int currentPageRowCount = 1; int num = 0; // 处理所有记录 for (int i = 0; i < records.size(); i++) { EntityItem record = records.get(i); // 分批处理:定期写入磁盘释放内存 if (i > 0 && i % BATCH_SIZE == 0) { doc.write(fos); fos.flush(); doc.removeBodyElement(doc.getPosOfTable(currentTable)); currentTable = createNewTable(doc); } // 计算图片行数 int maxImageRows = calculateMaxImageRows( record.getPropertyUri(), record.getNameplateUri(), record.getProductUri() ); // 分页检查 if (currentPageRowCount + maxImageRows > PAGE_SIZE_LIMIT) { doc.createParagraph().setPageBreak(true); currentTable = createNewTable(doc); currentPageRowCount = 1; } // 创建数据行 XWPFTableRow row = currentTable.createRow(); row.setCantSplitRow(true); // 文本列 createFixedWidthTextCell(row, 0, String.valueOf(++num), 800); createFixedWidthTextCell(row, 1, record.getToolNumber(), 1000); createFixedWidthTextCell(row, 2, record.getToolName(), 1000); // 图片列(使用流式处理) addImagesToCellWithStream(row.getCell(3), record.getPropertyUri(), 6800, 2, 1); addImagesToCellWithStream(row.getCell(4), record.getNameplateUri(), 3000, 1, 2); addImagesToCellWithStream(row.getCell(5), record.getProductUri(), 3000, 2, 3); currentPageRowCount += maxImageRows; } // 添加页码并写入最终内容 addPageNumbers(doc); doc.write(fos); // 从临时文件读取到字节流 return readTempFileToOutputStream(tempFile); } finally { // 确保删除临时文件 if (tempFile.exists()) { tempFile.delete(); } } } // 读取临时文件到字节流 private ByteArrayOutputStream readTempFileToOutputStream(File tempFile) throws IOException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); FileInputStream fis = new FileInputStream(tempFile)) { byte[] buffer = new byte[1024]; int len; while ((len = fis.read(buffer)) > 0) { outputStream.write(buffer, 0, len); } return outputStream; } } // 统一表格创建方法 private XWPFTable createNewTable(XWPFDocument doc) { XWPFTable table = doc.createTable(); setupTable(table); createTableHeader(table); return table; } // 优化图片行数计算 private int calculateMaxImageRows(List<FileRecords>... imageLists) { int maxRows = 0; for (List<FileRecords> list : imageLists) { if (list != null && !list.isEmpty()) { int itemCount = list.size(); int cols = (list == imageLists[2]) ? 4 : 2; // 产品图4列,其它2列 int rows = (int) Math.ceil((double) itemCount / cols); maxRows = Math.max(maxRows, rows); } } return maxRows > 0 ? maxRows : 1; // 至少1行 } // 页面设置 private static void setupPageLayout(XWPFDocument doc) { CTSectPr sectPr = doc.getDocument().getBody().addNewSectPr(); CTPageSz pageSize = sectPr.addNewPgSz(); pageSize.setOrient(STPageOrientation.LANDSCAPE); pageSize.setW(BigInteger.valueOf(16840)); pageSize.setH(BigInteger.valueOf(11900)); CTPageMar pageMar = sectPr.addNewPgMar(); pageMar.setLeft(BigInteger.valueOf(390)); pageMar.setRight(BigInteger.valueOf(390)); pageMar.setTop(BigInteger.valueOf(900)); pageMar.setBottom(BigInteger.valueOf(900)); pageMar.setFooter(BigInteger.valueOf(900)); } // 表格设置 private static void setupTable(XWPFTable table) { table.setWidth("100%"); int[] columnWidths = {500, 500, 500, 5000, 2000, 5000}; CTTbl cttbl = table.getCTTbl(); CTTblGrid tblGrid = cttbl.getTblGrid() != null ? cttbl.getTblGrid() : cttbl.addNewTblGrid(); // 清空现有网格列 if (tblGrid.sizeOfGridColArray() > 0) { tblGrid.removeGridCol(0); } // 添加新网格列 for (int width : columnWidths) { CTTblGridCol gridCol = tblGrid.addNewGridCol(); gridCol.setW(BigInteger.valueOf(width)); } } // 创建表头 private static void createTableHeader(XWPFTable table) { XWPFTableRow headerRow = table.getRow(0); for (int i = 0; i < 6; i++) { XWPFTableCell cell = headerRow.getCell(i) != null ? headerRow.getCell(i) : headerRow.createCell(); // 清除现有内容 for (int j = cell.getParagraphs().size() - 1; j >= 0; j--) { cell.removeParagraph(j); } // 添加新内容 XWPFParagraph para = cell.addParagraph(); para.setAlignment(ParagraphAlignment.CENTER); XWPFRun run = para.createRun(); run.setBold(true); run.setFontSize(12); run.setText(getHeaderText(i)); } } // 固定宽度文本单元格 private void createFixedWidthTextCell(XWPFTableRow row, int pos, String text, int width) { XWPFTableCell cell = row.getCell(pos) != null ? row.getCell(pos) : row.createCell(); // 清除现有内容 for (int i = cell.getParagraphs().size() - 1; i >= 0; i--) { cell.removeParagraph(i); } // 设置固定宽度 CTTcPr tcPr = cell.getCTTc().getTcPr(); if (tcPr == null) tcPr = cell.getCTTc().addNewTcPr(); tcPr.addNewTcW().setW(BigInteger.valueOf(width)); // 添加内容 XWPFParagraph para = cell.addParagraph(); para.setAlignment(ParagraphAlignment.CENTER); para.setWordWrap(true); XWPFRun run = para.createRun(); run.setText(text); } // 图片流获取方法 private InputStream getImageStream(String uri) throws IOException { URL url = new URL(uri); URLConnection conn = url.openConnection(); conn.setConnectTimeout(5000); conn.setReadTimeout(10000); return conn.getInputStream(); } // 图片尺寸计算 private int[] calculateImageDimensions(int flags) { switch (flags) { case 1: return new int[]{135, 120}; // 属性图 case 2: return new int[]{245, 120}; // 铭牌 case 3: return new int[]{60, 60}; // 产品图 default: return new int[]{120, 120};// 默认 } } // 表头文本 private static String getHeaderText(int colIndex) { String[] headers = {"序号", "工装编号", "资产名称", "关联图片", "铭牌", "产品图片"}; return colIndex < headers.length ? headers[colIndex] : ""; } // 添加页码 private static void addPageNumbers(XWPFDocument doc) { XWPFFooter footer = doc.createFooter(HeaderFooterType.DEFAULT); XWPFParagraph para = footer.createParagraph(); para.setAlignment(ParagraphAlignment.CENTER); XWPFRun run = para.createRun(); run.getCTR().addNewFldChar().setFldCharType(STFldCharType.BEGIN); run.getCTR().addNewInstrText().setStringValue("PAGE"); run.getCTR().addNewFldChar().setFldCharType(STFldCharType.END); run.setText(" / "); run.getCTR().addNewFldChar().setFldCharType(STFldCharType.BEGIN); run.getCTR().addNewInstrText().setStringValue("NUMPAGES"); run.getCTR().addNewFldChar().setFldCharType(STFldCharType.END); } // 图片类型检测 private int determineImageType(String fileName) { if (fileName == null || fileName.lastIndexOf(".") == -1) { return XWPFDocument.PICTURE_TYPE_PNG; // 默认PNG } String ext = fileName.substring(fileName.lastIndexOf(".") + 1).toLowerCase(); switch (ext) { case "png": return XWPFDocument.PICTURE_TYPE_PNG; case "jpg": case "jpeg": return XWPFDocument.PICTURE_TYPE_JPEG; case "gif": return XWPFDocument.PICTURE_TYPE_GIF; case "bmp": return XWPFDocument.PICTURE_TYPE_BMP; default: return XWPFDocument.PICTURE_TYPE_PNG; } } // 修改图片处理方法 private void addImagesToCellWithStream(XWPFTableCell cell, List<FileRecords> imagePaths, int cellWidth, int colsPerRow, int flags) { if (cell == null || imagePaths == null || imagePaths.isEmpty()) return; // 清除单元格现有内容 for (int i = cell.getParagraphs().size() - 1; i >= 0; i--) { cell.removeParagraph(i); } // 设置单元格宽度 CTTcPr tcPr = cell.getCTTc().getTcPr(); if (tcPr == null) tcPr = cell.getCTTc().addNewTcPr(); tcPr.addNewTcW().setW(BigInteger.valueOf(cellWidth)); XWPFParagraph currentPara = cell.addParagraph(); currentPara.setAlignment(ParagraphAlignment.LEFT); XWPFRun currentRun = currentPara.createRun(); for (int i = 0; i < imagePaths.size(); i++) { FileRecords imagePath = imagePaths.get(i); try { // 获取优化后的图片流 try (InputStream is = getOptimizedImageStream(imagePath.getFileUri(), flags)) { if (is != null) { int imageType = determineImageType(imagePath.getFileUri()); int[] dimensions = calculateImageDimensions(flags); // 添加压缩后的图片 currentRun.addPicture( is, imageType, imagePath.getFileUri(), Units.toEMU(dimensions[0]), Units.toEMU(dimensions[1]) ); } } // 换行控制 if ((i + 1) % colsPerRow == 0 && (i + 1) < imagePaths.size()) { currentPara = cell.addParagraph(); currentPara.setAlignment(ParagraphAlignment.LEFT); currentRun = currentPara.createRun(); } } catch (Exception e) { currentRun.setText("[图片处理失败: " + e.getMessage() + "]"); } } } // 图片优化处理 private InputStream getOptimizedImageStream(String uri, int flags) throws IOException { URL url = new URL(uri); URLConnection conn = url.openConnection(); conn.setConnectTimeout(5000); conn.setReadTimeout(10000); try (InputStream originalStream = conn.getInputStream()) { // 获取原始图片 BufferedImage originalImage = ImageIO.read(originalStream); if (originalImage == null) return null; // 计算目标尺寸并缩放 int[] targetDimensions = calculateTargetDimensions(originalImage, flags); BufferedImage scaledImage = scaleImage(originalImage, targetDimensions[0], targetDimensions[1]); // 压缩并转换格式 return compressImage(scaledImage, MAX_IMAGE_SIZE_KB * 1024); } } // 计算优化后的尺寸 private int[] calculateTargetDimensions(BufferedImage image, int flags) { int[] displayDims = calculateImageDimensions(flags); int displayWidth = displayDims[0]; int displayHeight = displayDims[1]; // 保持原始宽高比 double aspectRatio = (double) image.getWidth() / image.getHeight(); int targetWidth = displayWidth; int targetHeight = (int) (targetWidth / aspectRatio); // 确保不小于显示尺寸 if (targetHeight < displayHeight) { targetHeight = displayHeight; targetWidth = (int) (targetHeight * aspectRatio); } // 限制最大尺寸(避免过大) int maxDimension = 1500; // 最大像素尺寸 if (targetWidth > maxDimension || targetHeight > maxDimension) { if (targetWidth > targetHeight) { targetWidth = maxDimension; targetHeight = (int) (maxDimension / aspectRatio); } else { targetHeight = maxDimension; targetWidth = (int) (maxDimension * aspectRatio); } } return new int[]{targetWidth, targetHeight}; } // 图片缩放 private BufferedImage scaleImage(BufferedImage original, int width, int height) { BufferedImage scaled = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); Graphics2D g = scaled.createGraphics(); g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR); g.drawImage(original, 0, 0, width, height, null); g.dispose(); return scaled; } // 图片压缩 private InputStream compressImage(BufferedImage image, long maxSize) throws IOException { float quality = 0.8f; // 初始质量 ByteArrayOutputStream output = new ByteArrayOutputStream(); // 迭代压缩直到满足大小 do { output.reset(); ImageWriter writer = ImageIO.getImageWritersByFormatName("jpg").next(); ImageWriteParam param = writer.getDefaultWriteParam(); // 设置压缩参数 if (param.canWriteCompressed()) { param.setCompressionMode(ImageWriteParam.MODE_EXPLICIT); param.setCompressionQuality(quality); } try (ImageOutputStream ios = ImageIO.createImageOutputStream(output)) { writer.setOutput(ios); writer.write(null, new IIOImage(image, null, null), param); } writer.dispose(); quality -= 0.1f; // 降低质量 } while (output.size() > maxSize && quality > 0.3f); // 转换为JPEG格式(更好的压缩率) return new ByteArrayInputStream(output.toByteArray()); }压缩图片有点太花了怎么办
最新发布
09-26
评论
成就一亿技术人!
拼手气红包6.0元
还能输入1000个字符
 
红包 添加红包
表情包 插入表情
 条评论被折叠 查看
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值