diff --git a/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java b/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java index 5be9ef1df..3ec090f57 100644 --- a/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java +++ b/modules/cluster/src/main/java/org/apache/fluo/cluster/runner/AppRunner.java @@ -209,14 +209,6 @@ public static class ScanOptions { + "internal schema, making it easier to comprehend.") public boolean scanAccumuloTable = false; - public boolean exportAsCsv = false; - public String csvHeader; - public String csvDelimiter; - public String csvEscape; - public String csvQuote; - public String csvQuoteMode; - public boolean exportAsJson = false; - public String getStartRow() { return startRow; } @@ -242,8 +234,7 @@ public List getColumns() { public ScanUtil.ScanOpts getScanOpts() { return new ScanUtil.ScanOpts(startRow, endRow, columns, exactRow, rowPrefix, help, - hexEncNonAscii, scanAccumuloTable, exportAsCsv, csvDelimiter, csvEscape, csvHeader, - csvQuote, csvQuoteMode, exportAsJson); + hexEncNonAscii, scanAccumuloTable, false); } } } diff --git a/modules/command/pom.xml b/modules/command/pom.xml index f9949a285..8c8bf469d 100644 --- a/modules/command/pom.xml +++ b/modules/command/pom.xml @@ -46,10 +46,6 @@ org.apache.accumulo accumulo-core - - org.apache.commons - commons-lang3 - org.apache.curator curator-framework diff --git a/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java b/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java index 9f4455ac5..bcd03c07a 100644 --- a/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java +++ b/modules/command/src/main/java/org/apache/fluo/command/FluoScan.java @@ -4,9 +4,9 @@ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance with the License. You may obtain a * copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under @@ -20,7 +20,6 @@ import java.util.List; import com.beust.jcommander.Parameter; -import org.apache.commons.lang3.StringUtils; import org.apache.fluo.api.config.FluoConfiguration; import org.apache.fluo.core.client.FluoAdminImpl; import org.apache.fluo.core.util.ScanUtil; @@ -56,31 +55,6 @@ public static class ScanOptions extends CommonOpts { + "internal schema, making it easier to comprehend.") public boolean scanAccumuloTable = false; - @Parameter(names = "--csv", help = true, - description = "Export key/values stored in Accumulo as CSV file. Uses Fluo application " - + "properties to configure the CSV format.") - public boolean exportAsCsv = false; - - @Parameter(names = "--csv-header", help = true, description = "Set header for \"true\".") - public String csvHeader; - - @Parameter(names = "--csv-delimiter", help = true, - description = "Configure delimiter to a designeted character.") - public String csvDelimiter; - - @Parameter(names = "--csv-escape", help = true, - description = "Configure escape to a designeted character.") - public String csvEscape; - - @Parameter(names = "--csv-quote", help = true, - description = "Configure quote to a designeted character.") - public String csvQuote; - - @Parameter(names = "--csv-quote-mode", help = true, - description = "Configure quote mode to a designeted mode. The possible " - + "modes are: ALL, ALL_NON_NULL, MINIMAL, NONE and NON_NUMERIC") - public String csvQuoteMode; - @Parameter(names = "--json", help = true, description = "Export key/values stored in Accumulo as JSON file.") public boolean exportAsJson = false; @@ -101,26 +75,6 @@ public String getRowPrefix() { return rowPrefix; } - public String getCsvHeader() { - return csvHeader; - } - - public String getCsvDelimiter() { - return csvDelimiter; - } - - public String getCsvEscape() { - return csvEscape; - } - - public String getCsvQuote() { - return csvQuote; - } - - public String getCsvQuoteMode() { - return csvQuoteMode; - } - public List getColumns() { if (columns == null) { return Collections.emptyList(); @@ -130,25 +84,17 @@ public List getColumns() { /** * Check if the parameters informed can be used together. - * @since 1.2 */ private void checkScanOptions() { - if (this.exportAsCsv && this.exportAsJson) { + if (this.scanAccumuloTable && this.exportAsJson) { throw new IllegalArgumentException( - "Both \"--csv\" and \"--json\" can not be set together."); - } - - if (!this.exportAsCsv && (StringUtils.isNotEmpty(this.csvDelimiter) - | StringUtils.isNotEmpty(this.csvEscape) | StringUtils.isNotEmpty(this.csvHeader) - | StringUtils.isNotEmpty(this.csvQuote) | StringUtils.isNotEmpty(this.csvQuoteMode))) { - throw new IllegalArgumentException("No \"--csv\" detected"); + "Both \"--raw\" and \"--json\" can not be set together."); } } public ScanUtil.ScanOpts getScanOpts() { return new ScanUtil.ScanOpts(startRow, endRow, columns, exactRow, rowPrefix, help, - hexEncNonAscii, scanAccumuloTable, exportAsCsv, csvDelimiter, csvEscape, csvHeader, - csvQuote, csvQuoteMode, exportAsJson); + hexEncNonAscii, scanAccumuloTable, exportAsJson); } public static ScanOptions parse(String[] args) { diff --git a/modules/core/pom.xml b/modules/core/pom.xml index 0cfc6b3e9..fdd1f695a 100644 --- a/modules/core/pom.xml +++ b/modules/core/pom.xml @@ -49,14 +49,6 @@ org.apache.accumulo accumulo-core - - org.apache.commons - commons-csv - - - org.apache.commons - commons-lang3 - org.apache.curator curator-client diff --git a/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java b/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java index 78c3efbb5..4690fd5ab 100644 --- a/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java +++ b/modules/core/src/main/java/org/apache/fluo/core/util/ScanUtil.java @@ -22,24 +22,13 @@ import java.util.Collections; import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.function.Function; -import com.google.common.collect.Iterables; -import com.google.gson.FieldNamingPolicy; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonIOException; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.Scanner; import org.apache.accumulo.core.security.Authorizations; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; -import org.apache.commons.csv.QuoteMode; -import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.ObjectUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.fluo.accumulo.format.FluoFormatter; import org.apache.fluo.api.client.FluoClient; import org.apache.fluo.api.client.FluoFactory; @@ -50,14 +39,14 @@ import org.apache.fluo.api.data.Column; import org.apache.fluo.api.data.RowColumnValue; import org.apache.fluo.api.data.Span; -import org.apache.fluo.api.exceptions.FluoException; + +import com.google.common.collect.Iterables; +import com.google.gson.FieldNamingPolicy; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonIOException; public class ScanUtil { - public static final String CSV_HEADER = "csv.header"; - public static final String CSV_QUOTE_MODE = "csv.quoteMode"; - public static final String CSV_QUOTE = "csv.quote"; - public static final String CSV_ESCAPE = "csv.escape"; - public static final String CSV_DELIMITER = "csv.delimiter"; public static final String FLUO_VALUE = "value"; public static final String FLUO_COLUMN_VISIBILITY = "visibility"; public static final String FLUO_COLUMN_QUALIFIER = "qualifier"; @@ -115,6 +104,15 @@ public static Collection getColumns(ScanOpts options) { return columns; } + + private static Function getEncoder(ScanOpts options) { + if (options.hexEncNonAscii) { + return Hex::encNonAscii; + } else { + return Bytes::toString; + } + } + public static void scanFluo(ScanOpts options, FluoConfiguration sConfig, PrintStream out) throws IOException { @@ -123,152 +121,56 @@ public static void scanFluo(ScanOpts options, FluoConfiguration sConfig, PrintSt Span span = getSpan(options); Collection columns = getColumns(options); + CellScanner cellScanner = s.scanner().over(span).fetch(columns).build(); + Function encoder = getEncoder(options); if (options.exportAsJson) { - generateJson(options, span, columns, s, out); - } else { // TSV or CSV format - generateTsvCsv(options, span, columns, s, out); - } - - } catch (FluoException e) { - throw e; - } - } - } - - /** - * Generate TSV or CSV format as result of the scan. - * - * @since 1.2 - */ - private static void generateTsvCsv(ScanOpts options, Span span, Collection columns, - final Snapshot snapshot, PrintStream out) throws IOException { - // CSV Formater - CSVFormat csvFormat = CSVFormat.DEFAULT; - csvFormat = csvFormat.withQuoteMode(QuoteMode.ALL); - csvFormat = csvFormat.withRecordSeparator("\n"); - - // when "--csv" parameter is passed the "fluo.scan.csv" is analised - if (options.exportAsCsv) { - if (StringUtils.isNotEmpty(options.csvDelimiter)) { - if (options.csvDelimiter.length() > 1) { - throw new IllegalArgumentException( - "Invalid character for the \"--csv-delimiter\" parameter."); - } - csvFormat = csvFormat.withDelimiter(options.csvDelimiter.charAt(0)); - } - - if (StringUtils.isNotEmpty(options.csvEscape)) { - if (options.csvEscape.length() > 1) { - throw new IllegalArgumentException( - "Invalid character for the \"--csv-escape\" parameter."); - } - csvFormat = csvFormat.withEscape(options.csvEscape.charAt(0)); - } - - if (StringUtils.isNotEmpty(options.csvQuote)) { - if (options.csvQuote.length() > 1) { - throw new IllegalArgumentException( - "Invalid character for the \"--csv-quote\" parameter."); - } - csvFormat = csvFormat.withQuote(options.csvQuote.charAt(0)); - } - - // It can throw "java.lang.IllegalArgumentException" if the value not exists - // in "org.apache.commons.csv.QuoteMode" - if (StringUtils.isNotEmpty(options.csvQuoteMode)) { - csvFormat = csvFormat.withQuoteMode(QuoteMode.valueOf(options.csvQuoteMode)); - } - - if (BooleanUtils.toBooleanObject( - ObjectUtils.defaultIfNull(options.csvHeader, Boolean.FALSE.toString()))) { - csvFormat = csvFormat.withHeader(FLUO_ROW, FLUO_COLUMN_FAMILY, FLUO_COLUMN_QUALIFIER, - FLUO_COLUMN_VISIBILITY, FLUO_VALUE); - } - } else { - // Default TAB separator and NO quotes if possible. - csvFormat = csvFormat.withDelimiter(CSVFormat.TDF.getDelimiter()); - csvFormat = csvFormat.withQuoteMode(QuoteMode.MINIMAL); - } - - try (CSVPrinter printer = new CSVPrinter(out, csvFormat)) { - CellScanner cellScanner = snapshot.scanner().over(span).fetch(columns).build(); - - List record = new LinkedList<>(); - StringBuilder sb = new StringBuilder(); - int lines2check = 0; - for (RowColumnValue rcv : cellScanner) { - record.clear(); - if (options.hexEncNonAscii) { - sb.setLength(0); - Hex.encNonAscii(sb, rcv.getRow()); - record.add(sb.toString()); - sb.setLength(0); - Hex.encNonAscii(sb, rcv.getColumn().getFamily()); - record.add(sb.toString()); - sb.setLength(0); - Hex.encNonAscii(sb, rcv.getColumn().getQualifier()); - record.add(sb.toString()); - sb.setLength(0); - Hex.encNonAscii(sb, rcv.getColumn().getVisibility()); - record.add(sb.toString()); - sb.setLength(0); - Hex.encNonAscii(sb, rcv.getValue()); - record.add(sb.toString()); + generateJson(cellScanner, encoder, out); } else { - record.add(rcv.getsRow()); - record.add(rcv.getColumn().getFamily()); - record.add(rcv.getColumn().getQualifier()); - record.add(rcv.getColumn().getVisibility()); - record.add(rcv.getsValue()); - } - - printer.printRecord(record); - lines2check++; - if (lines2check == 100) { - lines2check = 0; - if (out.checkError()) { - throw new IOException("Fail to write data to stream."); + for (RowColumnValue rcv : cellScanner) { + out.print(encoder.apply(rcv.getRow())); + out.print(' '); + out.print(encoder.apply(rcv.getColumn().getFamily())); + out.print(' '); + out.print(encoder.apply(rcv.getColumn().getQualifier())); + out.print(' '); + out.print(encoder.apply(rcv.getColumn().getVisibility())); + out.print("\t"); + out.print(encoder.apply(rcv.getValue())); + out.println(); + if (out.checkError()) { + break; + } } } } } - out.flush(); } /** * Generate JSON format as result of the scan. - * + * * @since 1.2 */ - private static void generateJson(ScanOpts options, Span span, Collection columns, - final Snapshot snapshot, PrintStream out) throws JsonIOException { + private static void generateJson(CellScanner cellScanner, Function encoder, + PrintStream out) throws JsonIOException { Gson gson = new GsonBuilder().serializeNulls().setDateFormat(DateFormat.LONG) .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES).setVersion(1.0) .create(); - CellScanner cellScanner = snapshot.scanner().over(span).fetch(columns).build(); - - StringBuilder sb = new StringBuilder(); Map json = new LinkedHashMap<>(); for (RowColumnValue rcv : cellScanner) { - sb.setLength(0); - Hex.encNonAscii(sb, rcv.getRow()); - json.put(FLUO_ROW, sb.toString()); - sb.setLength(0); - Hex.encNonAscii(sb, rcv.getColumn().getFamily()); - json.put(FLUO_COLUMN_FAMILY, sb.toString()); - Hex.encNonAscii(sb, rcv.getColumn().getQualifier()); - json.put(FLUO_COLUMN_QUALIFIER, sb.toString()); - sb.setLength(0); - Hex.encNonAscii(sb, rcv.getColumn().getVisibility()); - json.put(FLUO_COLUMN_VISIBILITY, sb.toString()); - sb.setLength(0); - Hex.encNonAscii(sb, rcv.getValue()); - json.put(FLUO_VALUE, sb.toString()); - + json.put(FLUO_ROW, encoder.apply(rcv.getRow())); + json.put(FLUO_COLUMN_FAMILY, encoder.apply(rcv.getColumn().getFamily())); + json.put(FLUO_COLUMN_QUALIFIER, encoder.apply(rcv.getColumn().getQualifier())); + json.put(FLUO_COLUMN_VISIBILITY, encoder.apply(rcv.getColumn().getVisibility())); + json.put(FLUO_VALUE, encoder.apply(rcv.getValue())); gson.toJson(json, out); out.append("\n"); + + if (out.checkError()) { + break; + } } out.flush(); } @@ -311,18 +213,11 @@ public static class ScanOpts { public boolean help; public boolean hexEncNonAscii = true; public boolean scanAccumuloTable = false; - public boolean exportAsCsv = false; public boolean exportAsJson = false; - public final String csvDelimiter; - public final String csvEscape; - public final String csvHeader; - public final String csvQuote; - public final String csvQuoteMode; public ScanOpts(String startRow, String endRow, List columns, String exactRow, String rowPrefix, boolean help, boolean hexEncNonAscii, boolean scanAccumuloTable, - boolean exportAsCsv, String csvDelimiter, String csvEscape, String csvHeader, - String csvQuote, String csvQuoteMode, boolean exportAsJson) { + boolean exportAsJson) { this.startRow = startRow; this.endRow = endRow; this.columns = columns; @@ -331,12 +226,6 @@ public ScanOpts(String startRow, String endRow, List columns, String exa this.help = help; this.hexEncNonAscii = hexEncNonAscii; this.scanAccumuloTable = scanAccumuloTable; - this.exportAsCsv = exportAsCsv; - this.csvDelimiter = csvDelimiter; - this.csvEscape = csvEscape; - this.csvHeader = csvHeader; - this.csvQuote = csvQuote; - this.csvQuoteMode = csvQuoteMode; this.exportAsJson = exportAsJson; } diff --git a/modules/distribution/src/main/lib/fetch.sh b/modules/distribution/src/main/lib/fetch.sh index 51a54f479..2dd523c07 100755 --- a/modules/distribution/src/main/lib/fetch.sh +++ b/modules/distribution/src/main/lib/fetch.sh @@ -62,8 +62,6 @@ extra) download commons-collections:commons-collections:jar:3.2.1 download commons-configuration:commons-configuration:jar:1.10 download commons-io:commons-io:jar:2.4 - download org.apache.commons:commons-csv:jar:1.5 - download org.apache.commons:commons-lang3:jar:3.7 download io.dropwizard.metrics:metrics-core:jar:3.1.1 download io.dropwizard.metrics:metrics-graphite:jar:3.1.1 download javax.inject:javax.inject:jar:1 diff --git a/pom.xml b/pom.xml index 7307daaab..c4f8f3e54 100644 --- a/pom.xml +++ b/pom.xml @@ -139,16 +139,6 @@ accumulo-test ${accumulo.version} - - org.apache.commons - commons-csv - 1.5 - - - org.apache.commons - commons-lang3 - 3.7 - org.apache.curator curator-client