@Override public String contents() throws IOException { if (contents != null) { return contents; } else { ByteArrayOutputStream result = new ByteArrayOutputStream(); try (InputStream inputStream = inputStream()) { byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; int length; while ((length = inputStream.read(buffer)) != -1) { result.write(buffer, 0, length); } } return result.toString(charset().name()); } }
@Override public InputStream inputStream() throws IOException { return contents != null ? new ByteArrayInputStream(contents.getBytes(charset())) : new BOMInputStream(Files.newInputStream(path()), ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE); }
@Override public void publish(ScannerReportWriter writer) { for (final DefaultInputFile inputFile : componentCache.allChangedFilesToPublish()) { File iofile = writer.getSourceFile(inputFile.scannerId()); try (OutputStream output = new BufferedOutputStream(new FileOutputStream(iofile)); InputStream in = inputFile.inputStream(); BufferedReader reader = new BufferedReader(new InputStreamReader(in, inputFile.charset()))) { writeSource(reader, output, inputFile.lines()); } catch (IOException e) { throw new IllegalStateException("Unable to store file source in the report", e); } } }
@Test public void should_detect_charset_from_BOM() { Path basedir = Paths.get("src/test/resources/org/sonar/scanner/scan/filesystem/"); assertThat(createInputFileWithMetadata(basedir.resolve("without_BOM.txt")).charset()) .isEqualTo(StandardCharsets.US_ASCII); assertThat(createInputFileWithMetadata(basedir.resolve("UTF-8.txt")).charset()) .isEqualTo(StandardCharsets.UTF_8); assertThat(createInputFileWithMetadata(basedir.resolve("UTF-16BE.txt")).charset()) .isEqualTo(StandardCharsets.UTF_16BE); assertThat(createInputFileWithMetadata(basedir.resolve("UTF-16LE.txt")).charset()) .isEqualTo(StandardCharsets.UTF_16LE); assertThat(createInputFileWithMetadata(basedir.resolve("UTF-32BE.txt")).charset()) .isEqualTo(MetadataGenerator.UTF_32BE); assertThat(createInputFileWithMetadata(basedir.resolve("UTF-32LE.txt")).charset()) .isEqualTo(MetadataGenerator.UTF_32LE); }
@Test public void use_default_charset_if_detection_fails() throws IOException { Path tempFile = temp.newFile().toPath(); byte invalidWindows1252 = (byte) 129; byte[] b = {(byte) 0xDF, (byte) 0xFF, (byte) 0xFF, invalidWindows1252}; FileUtils.writeByteArrayToFile(tempFile.toFile(), b); DefaultInputFile inputFile = createInputFileWithMetadata(tempFile); assertThat(inputFile.charset()).isEqualTo(StandardCharsets.US_ASCII); }
@Before public void setUp() throws Exception { lastSnapshots = mock(ServerLineHashesLoader.class); file = mock(DefaultInputFile.class); ioFile = temp.newFile(); when(file.file()).thenReturn(ioFile); when(file.path()).thenReturn(ioFile.toPath()); when(file.inputStream()).thenAnswer(i -> Files.newInputStream(ioFile.toPath())); when(file.lines()).thenReturn(1); when(file.charset()).thenReturn(StandardCharsets.UTF_8); project = mock(DefaultInputProject.class); sourceHashHolder = new SourceHashHolder(project, file, lastSnapshots); }
@Test public void test_content_exclude_bom() throws IOException { Path testFile = baseDir.resolve(PROJECT_RELATIVE_PATH); Files.createDirectories(testFile.getParent()); try (BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(testFile.toFile()), StandardCharsets.UTF_8))) { out.write('\ufeff'); } String content = "test é string €"; Files.write(testFile, content.getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); assertThat(Files.readAllLines(testFile, StandardCharsets.UTF_8).get(0)).hasSize(content.length() + 1); Metadata metadata = new Metadata(42, 30, "", new int[0], new int[0], 10); DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> f.setMetadata(metadata)) .setStatus(InputFile.Status.ADDED) .setCharset(StandardCharsets.UTF_8); assertThat(inputFile.contents()).isEqualTo(content); try (InputStream inputStream = inputFile.inputStream()) { String result = new BufferedReader(new InputStreamReader(inputStream, inputFile.charset())).lines().collect(Collectors.joining()); assertThat(result).isEqualTo(content); } }
@Test public void test_content() throws IOException { Path testFile = baseDir.resolve(PROJECT_RELATIVE_PATH); Files.createDirectories(testFile.getParent()); String content = "test é string"; Files.write(testFile, content.getBytes(StandardCharsets.ISO_8859_1)); assertThat(Files.readAllLines(testFile, StandardCharsets.ISO_8859_1).get(0)).hasSize(content.length()); Metadata metadata = new Metadata(42, 30, "", new int[0], new int[0], 10); DefaultInputFile inputFile = new DefaultInputFile(indexedFile, f -> f.setMetadata(metadata)) .setStatus(InputFile.Status.ADDED) .setCharset(StandardCharsets.ISO_8859_1); assertThat(inputFile.contents()).isEqualTo(content); try (InputStream inputStream = inputFile.inputStream()) { String result = new BufferedReader(new InputStreamReader(inputStream, inputFile.charset())).lines().collect(Collectors.joining()); assertThat(result).isEqualTo(content); } }
@Test public void test() throws Exception { Metadata metadata = new Metadata(42, 42, "", new int[0], new int[0], 10); DefaultInputFile inputFile = new DefaultInputFile(indexedFile, (f) -> f.setMetadata(metadata)) .setStatus(InputFile.Status.ADDED) .setCharset(StandardCharsets.ISO_8859_1); assertThat(inputFile.absolutePath()).endsWith("Foo.php"); assertThat(inputFile.filename()).isEqualTo("Foo.php"); assertThat(inputFile.uri()).hasPath(baseDir.resolve(PROJECT_RELATIVE_PATH).toUri().getPath()); assertThat(new File(inputFile.absolutePath())).isAbsolute(); assertThat(inputFile.language()).isEqualTo("php"); assertThat(inputFile.status()).isEqualTo(InputFile.Status.ADDED); assertThat(inputFile.type()).isEqualTo(InputFile.Type.TEST); assertThat(inputFile.lines()).isEqualTo(42); assertThat(inputFile.charset()).isEqualTo(StandardCharsets.ISO_8859_1); assertThat(inputFile.getModuleRelativePath()).isEqualTo(MODULE_RELATIVE_PATH); assertThat(inputFile.getProjectRelativePath()).isEqualTo(PROJECT_RELATIVE_PATH); sensorStrategy.setGlobal(false); assertThat(inputFile.relativePath()).isEqualTo(MODULE_RELATIVE_PATH); assertThat(new File(inputFile.relativePath())).isRelative(); sensorStrategy.setGlobal(true); assertThat(inputFile.relativePath()).isEqualTo(PROJECT_RELATIVE_PATH); assertThat(new File(inputFile.relativePath())).isRelative(); }
assertThat(loadedFile.charset()).isEqualTo(StandardCharsets.UTF_8);
@Override public String contents() throws IOException { if (contents != null) { return contents; } else { ByteArrayOutputStream result = new ByteArrayOutputStream(); try (InputStream inputStream = inputStream()) { byte[] buffer = new byte[DEFAULT_BUFFER_SIZE]; int length; while ((length = inputStream.read(buffer)) != -1) { result.write(buffer, 0, length); } } return result.toString(charset().name()); } }
@Override public InputStream inputStream() throws IOException { return contents != null ? new ByteArrayInputStream(contents.getBytes(charset())) : new BOMInputStream(Files.newInputStream(path()), ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE); }
@Override public void publish(ScannerReportWriter writer) { for (final DefaultInputFile inputFile : componentCache.allChangedFilesToPublish()) { File iofile = writer.getSourceFile(inputFile.scannerId()); try (OutputStream output = new BufferedOutputStream(new FileOutputStream(iofile)); InputStream in = inputFile.inputStream(); BufferedReader reader = new BufferedReader(new InputStreamReader(in, inputFile.charset()))) { writeSource(reader, output, inputFile.lines()); } catch (IOException e) { throw new IllegalStateException("Unable to store file source in the report", e); } } }
/** * Compute a MD5 hash of each line of the file after removing of all blank chars */ public static void computeLineHashesForIssueTracking(DefaultInputFile f, LineHashConsumer consumer) { scanFile(f.file(), f.charset(), new LineHashComputer(consumer)); } }
@Override public void publish(BatchReportWriter writer) { for (final BatchComponent resource : resourceCache.all()) { if (!resource.isFile()) { continue; } DefaultInputFile inputFile = (DefaultInputFile) resource.inputComponent(); File iofile = writer.getSourceFile(resource.batchId()); int line = 0; try (FileOutputStream output = new FileOutputStream(iofile); BOMInputStream bomIn = new BOMInputStream(new FileInputStream(inputFile.file()), ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE); BufferedReader reader = new BufferedReader(new InputStreamReader(bomIn, inputFile.charset()))) { String lineStr = reader.readLine(); while (lineStr != null) { IOUtils.write(lineStr, output, StandardCharsets.UTF_8); line++; if (line < inputFile.lines()) { IOUtils.write("\n", output, StandardCharsets.UTF_8); } lineStr = reader.readLine(); } } catch (IOException e) { throw new IllegalStateException("Unable to store file source in the report", e); } } } }
FileSourceDb.Data.Builder createForSource(DefaultInputFile inputFile) throws IOException { FileSourceDb.Data.Builder result = FileSourceDb.Data.newBuilder(); List<String> lines = FileUtils.readLines(inputFile.file(), inputFile.charset()); // Missing empty last line if (lines.size() == inputFile.lines() - 1) { lines.add(""); } for (int lineIdx = 1; lineIdx <= lines.size(); lineIdx++) { String s = CharMatcher.anyOf(BOM).removeFrom(lines.get(lineIdx - 1)); FileSourceDb.Line.Builder linesBuilder = result.addLinesBuilder(); linesBuilder.setLine(lineIdx).setSource(s); } return result; }
void applyHighlighting(DefaultInputFile inputFile, InputFileMetadata metadata, FileSourceDb.Data.Builder to) { SyntaxHighlightingData highlighting = componentDataCache.getData(inputFile.key(), SnapshotDataTypes.SYNTAX_HIGHLIGHTING); String language = inputFile.language(); if (highlighting == null && language != null) { highlighting = codeColorizers.toSyntaxHighlighting(inputFile.file(), inputFile.charset(), language); } if (highlighting == null) { return; } StringBuilder[] highlightingPerLine = new StringBuilder[inputFile.lines()]; RuleItemWriter ruleItemWriter = new RuleItemWriter(); int currentLineIdx = 1; for (SyntaxHighlightingRule rule : highlighting.syntaxHighlightingRuleSet()) { while (currentLineIdx < inputFile.lines() && rule.getStartPosition() >= metadata.originalLineOffsets()[currentLineIdx]) { // This rule starts on another line so advance currentLineIdx++; } // Now we know current rule starts on current line writeDataPerLine(metadata.originalLineOffsets(), rule, rule.getStartPosition(), rule.getEndPosition(), highlightingPerLine, currentLineIdx, ruleItemWriter); } for (int i = 0; i < highlightingPerLine.length; i++) { StringBuilder sb = highlightingPerLine[i]; if (sb != null) { to.getLinesBuilder(i).setHighlighting(sb.toString()); } } }