merge2er

merge2er --help

Usage: merge2er [Options] [INPUT...]

INPUT                           Files or directories input location path

Options:

-h, --help

--version

# schema

-j, --job ID                    Job ID

# spark-submit

--master MASTER_URL             spark://host:port, mesos://host:port, yarn, or local.

--name NAME                     The name of your application.

--jars JARS                     Comma-separated list of local jars to include on the driver

--files FILES                   Comma-separated list of local files to include on the driver

--conf PROP=VALUE               Arbitrary Spark configuration property.

-b, --blocksize BLOCKSIZE       Block Size, eg. 1024kb, 64mb, 128mb...

-D, --warehouse-dir DIRECTORY   Spark SQL warehouse directory

--metrics DURATION              Enabling metrics printing in defined interval: 1h, 1m, 1s...

# input data

--manifest PATH                 Path to manifest file which contains 3 possible CSV formats:
                                * 1 column => <FilePath>
                                * 2 columns => <Bucket>,<FilePath>
                                * 3 columns => <Bucket>,<FilePath>,<Version>
                                The <FilePath> and <Bucket> are related to the manifest path filesystem and location

-U, --in-user USER              Input user

-P, --in-password PASSWORD      Input password

--in-token TOKEN                Session token of input S3 bucket

--in-key KEY                    Input authentication private key

--in-key-file FILE              Input authentication private key file

--in-key-password PASSWORD      Input authentication private key password

-F, --in-format FORMAT          Input format.
                                - jdbc, parquet, orc, json, csv, tsv, psv, avro...

--in-schema SCHEMA              Input schema related to hive, jdbc or spark connector sources

--schema-pattern PATTERN        Input schema pattern related hive, jdbc or spark connector sources: Ex "*word*"

--table-pattern PATTERN         Input table pattern related to hive, jdbc or spark connector sources: Ex "*word*"

--file-table, -q                Enables loading files as tables, instead of their directories as tables
-Q                              Disables loading file as tables.

-T, --table TABLE|QUERY         Name of the table (or specified SQL)

-I, --id-column <NAME|EXPR>[:DATATYPE] [EXTRA] [AS ALIAS]
                                Name of identifier column to contain document's ID values for tabular
                                files, jdbc or hive tables

-<MIN>, --id-min MIN            ID value specifying lower bound data range to be processed

+<MAX>, --id-max MAX            ID value specifying upper bound data range to be processed

--in-column-type [[SCHEMA.]TABLE.]COLUMN=DATA_TYPE
                                Enforces the data type of the matched columns when the tables are loaded,
                                instead of the inferred one (ex: csv/tsv/psv columns are strings by default).
                                The path is a glob: ** matches anything (crossing dots), * and ? match within
                                a single segment. The path to match depends on the loaded tables:
                                  - COLUMN: single table
                                  - TABLE.COLUMN: multiple tables or a single schema
                                  - SCHEMA.TABLE.COLUMN: multiple schemas
                                Can be repeated. Ex: "**.??code=DECIMAL(38,18)", "tab.col=BIGINT"

--in-column-type-pattern [[SCHEMA.]TABLE.]COLUMN=DATA_TYPE
                                Same as --in-column-type, but the path is a regular expression that must match
                                the whole column path. Ex: "\w+\.\w+\.codecode=DECIMAL(38,18)"

--where WHERE                   Where clause for filtering tabular files, jdbc or hive tables

--limit LIMIT                   Limit clause for filtering the number of records/documents

--has-header                    If the input csv/psv/tsv file has a header
                                default: true

--in-opt PROP=VALUE,...         Extra options for Spark dataFrame reader, ex: prop1=value1,prop2=value2

# metadata

-m, --meta URL                  JDBC location of metadata store
                                default: jdbc:postgresql:x2er

-M, --meta-user USER            Metadata user
                                default: flex2er

-w, --meta-password PASSWORD    Metadata password

--store-credentials             Enable storing credentials at metadata store

# output data

-o, --out OUTPUT                Output location path

-u, --out-user USER             Output user

-p, --out-password PASSWORD     Output password

--out-token TOKEN               Session token of output S3 bucket

--out-key KEY                   Output authentication private key

--out-key-file FILE             Output authentication private key file

--out-key-password PASSWORD     Output authentication private key password

-B, --batchsize BATCHSIZE       Batch size to write into databases
                                default: 1000

-f, --out-format FORMAT         Output format:
                                - jdbc, parquet, orc, json, csv, tsv, psv, avro...

-z, --compression COMPRESSION   Compression mode for filesystem output
                                - none, snappy, gzip, bzip2, deflate, lzo, lz4, zlib, xz...
                                default: snappy

-S, --savemode SAVEMODE         Save Mode when target table, directory or file exists
                                ex: [e]rror, [a]ppend, [o]verwrite, [i]gnore, [p]rintschema, [w]riteschema
                                default: append

-Y, --out-part PARTITIONS       Number of partitions for writing data

-V, --hive-create               Enable creating hive tables

-E, --out-schema SCHEMA         Creating hive or jdbc tables into schema

-r, --out-prefix PREFIX         Append a prefix in each output table name

-y, --out-suffix SUFFIX         Append a suffix in each output table name

--constraints                   Enables copying the primary and foreign key constraints

--comment-len LENGTH            Defining a length for table/column comments different from the output dialect

-R, --extra-column <NAME|EXPR>[:DATATYPE] [AUTO|RANDOM|NOW|ID|FILENAME|FILEPATH|=VALUE] [AS ALIAS]
                                Extra column(s) to be included in all output tables

-i, --partition-column <NAME|EXPR>[:DATATYPE] [AUTO|RANDOM|NOW|ID|FILENAME|FILEPATH|=VALUE] [AS ALIAS]
                                Partition column(s) to be included in all output tables

--rename [IDX=|NAME=]NEW_NAME,* Rename a table or multiple tables by index (1,2,3...) or by its original name

--out-opt PROP=VALUE,...        Extra options for Spark dataFrame writer, ex: prop1=value1,prop2=value2

--legacy-dialect-version VER    Changing the version of Flexter output dialect compatibility to keep older tables
                                compatible with newer Flexter version
                                default: 2

--namemode MODE                 Change the table/column original names to:
                                  - "": Keep original names
                                  - [l]ower: to lower case.
                                  - [u]pper: to upper case.
                                  - [c]camelcase: Remove separators and keep words separated by case.

--name-max-len SIZE             Maximum column name length
                                default: 30

--out-column-type [[SCHEMA.]TABLE.]COLUMN=DATA_TYPE
                                Converts the matched columns to the data type after the tables are loaded and
                                before they are written. Same path and glob rules as --in-column-type.
                                Can be repeated. Ex: "**.amount=DECIMAL(38,18)"

--out-column-type-pattern [[SCHEMA.]TABLE.]COLUMN=DATA_TYPE
                                Same as --out-column-type, but the path is a regular expression that must match
                                the whole column path. Ex: ".*\.amount=DECIMAL(38,18)"

# actions

--tz TIMEZONE                   Timezone for both JVM and Spark session, ex: UTC

--tolerance-level LEVEL         Table failures tolerance level:
                                  - 0 or [z]ero: Interrupts the job at the first table that fails to be written
                                  - 1 or [d]ocument: Skips the tables that fail to be written, keeping processing the other tables
                                  - 2 or [g]raceful: Same as document for merge2er
                                The job fails when no table could be written.
                                default: graceful

-c, --commands                  Show SQL commands

-s, --skip                      Skip writing results