database

package

v1.5.2Latest Latest

This package is not in the latest version of its module.

Go to latest Published: Dec 20, 2025 License:GPL-3.0Imports:123Imported by:3

Details

Valid go.mod file
Redistributable license
Tagged version
Stable version
Learn more about best practices

Repository

github.com/slingdata-io/sling-cli

Documentation ¶

Rendered for

Index¶

Constants¶

View Source

const RelationManyToMany = "many_to_many"

View Source

const RelationManyToOne = "many_to_one"

View Source

const RelationOneToMany = "one_to_many"

View Source

const RelationOneToOne = "one_to_one"

Variables¶

View Source

var ChunkByColumn = func(connConnection, tableTable, cstring, pint) ([]Table,error) {return []Table{table},g.Error("please use the official sling-cli release for chunking columns")}

View Source

var ChunkByColumnRange = func(connConnection, tTable, cstring, cs, min, maxstring) ([]Chunk,error) {return []Chunk{},g.Error("please use the official sling-cli release for chunking")}

View Source

var ChunkByCount = func(connConnection, tTable, cstring, ccint, min, maxstring) ([]Chunk,string,error) {return []Chunk{}, "",g.Error("please use the official sling-cli release for chunking")}

View Source

var ChunkByExpression = func(connConnection, tTable, estring, ccint) ([]Chunk,error) {return []Chunk{},g.Error("please use the official sling-cli release for chunking")}

View Source

var Debug =false

Debug prints queries when true

View Source

var DuckDbFileCmd = map[string]*exec.Cmd{}

View Source

var DuckDbFileContext = map[string]*g.Context{}// so that collision doesn't happen

View Source

var DuckDbMux =sync.Mutex{}

View Source

var DuckDbUseTempFile =false

View Source

var InferDBStream =false

InferDBStream may need to be `true`, since precision and scale is not guaranteed.If `false`, will use the database stream source schema

View Source

var SQLiteVersion = "3.41"

View Source

var (// UseBulkExportFlowCSV to use BulkExportFlowCSVUseBulkExportFlowCSV =false)

Functions¶

funcChangeColumnTypeViaAdd ¶

func ChangeColumnTypeViaAdd(connConnection, tableTable, coliop.Column) (errerror)

ChangeColumnTypeViaAdd swaps a new column with the old in order to change the typeneed to use this with snowflake when changing from date to string, or number to string

funcCopyFromAzure ¶

func CopyFromAzure(connConnection, tableFName, azPathstring) (errerror)

CopyFromAzure uses the Snowflake COPY INTO Table command from Azurehttps://docs.snowflake.com/en/sql-reference/sql/copy-into-table.html

funcCopyFromS3 ¶

func CopyFromS3(connConnection, tableFName, s3Pathstring) (errerror)

funcEnsureBinSQLite ¶

func EnsureBinSQLite() (binPathstring, errerror)

EnsureBinSQLite ensures sqlite binary existsif missing, downloads and uses

funcGenerateAlterDDL ¶

func GenerateAlterDDL(connConnection, tableTable, newColumnsiop.Columns) (bool,error)

GenerateAlterDDL generate a DDL based on a dataset

funcGetArrowDBCDriverType ¶added inv1.5.2

func GetArrowDBCDriverType(driverNamestring)dbio.Type

GetArrowDBCDriverType maps ADBC driver names to corresponding database typesThis allows using driver-specific SQL templates

funcGetOptimizeTableStatements ¶added inv1.1.8

func GetOptimizeTableStatements(connConnection, table *Table, newColumnsiop.Columns, isTempbool) (okbool, ddlParts []string, errerror)

GetOptimizeTableStatements analyzes the table and alters the table withthe columns data type based on its analysis resultif table is missing, it is created with a new DDlHole in this: will truncate data points, since it is basedonly on new data being inserted... would need a completestats of the target table to properly optimize.

funcGetQualifierQuote ¶

func GetQualifierQuote(dialectdbio.Type)string

funcInsertBatchStream ¶

func InsertBatchStream(connConnection, txTransaction, tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertBatchStream inserts a stream into a table in batch

funcInsertStream ¶

func InsertStream(connConnection, tx *BaseTransaction, tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertStream inserts a stream

funcMakeDuckDbSecretProps ¶added inv1.4.11

func MakeDuckDbSecretProps(connConnection, secretTypeiop.DuckDbSecretType) (secretProps map[string]string)

MakeDuckDbSecretProps convert conn props to duckdb secret props

funcMerge ¶added inv1.4.20

func Merge(connConnection, txTransaction, sourceTable, targetTablestring, pkFields []string) (countint64, errerror)

Merge upserts from source table into target table

funcNativeTypeToGeneral ¶

func NativeTypeToGeneral(name, dbTypestring, connConnection) (colTypeiop.ColumnType)

funcPK ¶

func PK(obj interface{}) (pk []string)

PK returns the primary keys of a model

funcParseColumnName ¶

func ParseColumnName(textstring, dialectdbio.Type) (colNamestring, errerror)

funcParseSQLMultiStatements ¶

func ParseSQLMultiStatements(sqlstring, Dialect ...dbio.Type) (sqls []string)

ParseSQLMultiStatements splits a sql text into statementstypically by a ';'

funcPrintSessionID ¶added inv1.4.24

func PrintSessionID(connConnection)

funcQueryOperation ¶added inv1.4.24

func QueryOperation(connConnection, operationOperation, params map[string]any) (querystring, errerror)

QueryOperation uses the operation input to render/generate a querybased on a template. This should be connection agnostic.

funcSQLColumns ¶

func SQLColumns(colTypes []ColumnType, connConnection) (columnsiop.Columns)

SQLColumns returns the columns from database ColumnType

funcSplitTableFullName ¶

func SplitTableFullName(tableNamestring) (string,string)

SplitTableFullName retrusn the schema / table name

funcTestPermissions ¶

func TestPermissions(connConnection, tableNamestring) (errerror)

TestPermissions tests the needed permissions in a given connection

funcTrimSQLComments ¶added inv1.4.7

func TrimSQLComments(sqlstring) (string,error)

TrimSQLComments removes all SQL comments (line and block comments) from the input SQLLine comments start with -- and end with a newlineBlock comments start with /* and end with */

funcTruncateTable ¶added inv1.4.14

func TruncateTable(connConnection, tableNamestring)error

funcUID ¶

func UID(obj interface{})string

UID returns a unique ID for that object

Types¶

typeArrowDBConn ¶added inv1.5.2

type ArrowDBConn struct {BaseConnURLstringConnadbc.Connection// contains filtered or unexported fields}

ArrowDBConn is an Arrow FlightSQL connection

func (*ArrowDBConn)BulkExportFlow ¶added inv1.5.2

func (conn *ArrowDBConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

BulkExportFlow exports data as a dataflow

func (*ArrowDBConn)BulkExportStream ¶added inv1.5.2

func (conn *ArrowDBConn) BulkExportStream(tableTable) (ds *iop.Datastream, errerror)

BulkExportStream streams the rows in bulk

func (*ArrowDBConn)BulkImportFlow ¶added inv1.5.2

func (conn *ArrowDBConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow imports data from a dataflow using ADBC bulk ingestion

func (*ArrowDBConn)BulkImportStream ¶added inv1.5.2

func (conn *ArrowDBConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream imports data from a datastream using ADBC bulk ingestion

func (*ArrowDBConn)Close ¶added inv1.5.2

func (conn *ArrowDBConn) Close()error

Close closes the ADBC connection and database

func (*ArrowDBConn)Connect ¶added inv1.5.2

func (conn *ArrowDBConn) Connect(timeOut ...int) (errerror)

Connect opens the ADBC connection

func (*ArrowDBConn)ExecContext ¶added inv1.5.2

func (conn *ArrowDBConn) ExecContext(ctxcontext.Context, sqlstring, args ...interface{}) (resultsql.Result, errerror)

ExecContext executes a SQL statement (read-only operations)

func (*ArrowDBConn)GetNativeType ¶added inv1.5.2

func (conn *ArrowDBConn) GetNativeType(coliop.Column) (nativeTypestring, errerror)

GetNativeType returns the native column type from generic

func (*ArrowDBConn)GetSQLColumns ¶added inv1.5.2

func (conn *ArrowDBConn) GetSQLColumns(tableTable) (columnsiop.Columns, errerror)

GetSQLColumns returns columns for a SQL query using Arrow schemaThis avoids wrapping with LIMIT which may not work for all database types

func (*ArrowDBConn)GetTemplateValue ¶added inv1.5.2

func (conn *ArrowDBConn) GetTemplateValue(pathstring)string

GetTemplateValue returns the template value for the given pathIt first checks the driver-specific template, then falls back to ADBC template

func (*ArrowDBConn)Init ¶added inv1.5.2

func (conn *ArrowDBConn) Init()error

Init initiates the connection

func (*ArrowDBConn)LoadTemplates ¶added inv1.5.2

func (conn *ArrowDBConn) LoadTemplates()error

LoadTemplates loads the appropriate yaml templateFor ADBC, it merges the driver-specific template with the ADBC templateDriver template is base, ADBC template overrides for ADBC-specific behavior

func (*ArrowDBConn)Quote ¶added inv1.5.2

func (conn *ArrowDBConn) Quote(fieldstring)string

func (*ArrowDBConn)StreamRowsContext ¶added inv1.5.2

func (conn *ArrowDBConn) StreamRowsContext(ctxcontext.Context, sqlstring, options ...map[string]interface{}) (ds *iop.Datastream, errerror)

StreamRowsContext streams query results as a datastream using Arrow record batches

func (*ArrowDBConn)Template ¶added inv1.5.2

func (conn *ArrowDBConn) Template()dbio.Template

func (*ArrowDBConn)Unquote ¶added inv1.5.2

func (conn *ArrowDBConn) Unquote(fieldstring)string

typeAthenaConn ¶added inv1.4.7

type AthenaConn struct {BaseConnClient          *athena.ClientURLstringDataLocationstringStagingLocationstring}

AthenaConn is an Athena connection

func (*AthenaConn)BulkExportFlow ¶added inv1.4.7

func (conn *AthenaConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

BulkExportFlow reads in bulk

func (*AthenaConn)BulkExportStream ¶added inv1.4.7

func (conn *AthenaConn) BulkExportStream(tableTable) (ds *iop.Datastream, errerror)

BulkExportStream reads in bulk

func (*AthenaConn)BulkImportFlow ¶added inv1.4.7

func (conn *AthenaConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow inserts a flow of streams into a table.For athena we need to create CSVs in S3 and then use the COPY command.

func (*AthenaConn)BulkImportStream ¶added inv1.4.7

func (conn *AthenaConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream inserts a stream into a table.For athena we need to create CSVs in S3 and then use the COPY command.

func (*AthenaConn)CastColumnForSelect ¶added inv1.4.7

func (conn *AthenaConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*AthenaConn)Close ¶added inv1.4.7

func (conn *AthenaConn) Close()error

Close closes the connection

func (*AthenaConn)Connect ¶added inv1.4.7

func (conn *AthenaConn) Connect(timeOut ...int) (errerror)

Connect connects to the database

func (*AthenaConn)ExecContext ¶added inv1.4.7

func (conn *AthenaConn) ExecContext(ctxcontext.Context, sqlstring, args ...interface{}) (resultsql.Result, errerror)

ExecContext executes the sql query

func (*AthenaConn)GenerateDDL ¶added inv1.4.7

func (conn *AthenaConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (sqlstring, errerror)

GenerateDDL generates a DDL based on a dataset

func (*AthenaConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *AthenaConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*AthenaConn)GetDatabases ¶added inv1.4.7

func (conn *AthenaConn) GetDatabases() (iop.Dataset,error)

GetDatabases returns databases for given connection

func (*AthenaConn)GetSchemata ¶added inv1.4.7

func (conn *AthenaConn) GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (Schemata,error)

GetSchemata obtain full schemata info for a schema and/or table in current database

func (*AthenaConn)Init ¶added inv1.4.7

func (conn *AthenaConn) Init()error

Init initiates the object

func (*AthenaConn)InsertBatchStream ¶added inv1.4.7

func (conn *AthenaConn) InsertBatchStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertBatchStream inserts a stream into a table in batch

func (*AthenaConn)InsertStream ¶added inv1.4.7

func (conn *AthenaConn) InsertStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertStream demonstrates loading data into a Athena table using a file on the local filesystem.

func (*AthenaConn)LoadFromS3 ¶added inv1.4.10

func (conn *AthenaConn) LoadFromS3(tableFName, s3Pathstring, columnsiop.Columns) (countuint64, errerror)

LoadFromS3 creates a temporary external table pointing to S3, then inserts into the target table

func (*AthenaConn)NewTransaction ¶added inv1.4.7

func (conn *AthenaConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

func (*AthenaConn)StagingBucket ¶added inv1.4.10

func (conn *AthenaConn) StagingBucket()string

func (*AthenaConn)StreamRowsContext ¶added inv1.4.7

func (conn *AthenaConn) StreamRowsContext(ctxcontext.Context, sqlstring, options ...map[string]interface{}) (ds *iop.Datastream, errerror)

StreamRowsContext executes the query and streams the result into a datastream

func (*AthenaConn)Unload ¶added inv1.4.7

func (conn *AthenaConn) Unload(ctx *g.Context, tables ...Table) (s3Pathstring, errerror)

Unload exports the sql query into an S3 bucket

typeAzureTableConn ¶added inv1.4.16

type AzureTableConn struct {BaseConnURLstringClient        *aztables.ServiceClientAccountNamestringAccountKeystringSASTokenstringConnectionStrstringTableNamestring// Specific table name for table-specific SAS tokens}

AzureTableConn is an Azure Table connection

func (*AzureTableConn)BulkExportFlow ¶added inv1.4.16

func (conn *AzureTableConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

func (*AzureTableConn)BulkImportFlow ¶added inv1.4.16

func (conn *AzureTableConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow imports data into Azure Tables

func (*AzureTableConn)Close ¶added inv1.4.16

func (conn *AzureTableConn) Close()error

func (*AzureTableConn)Connect ¶added inv1.4.16

func (conn *AzureTableConn) Connect(timeOut ...int)error

Connect connects to the database

func (*AzureTableConn)ExecContext ¶added inv1.4.16

func (conn *AzureTableConn) ExecContext(ctxcontext.Context, sqlstring, args ...any) (resultsql.Result, errerror)

func (*AzureTableConn)GetCount ¶added inv1.4.16

func (conn *AzureTableConn) GetCount(tableFNamestring) (int64,error)

GetCount returns the row count for a table

func (*AzureTableConn)GetSchemas ¶added inv1.4.16

func (conn *AzureTableConn) GetSchemas() (dataiop.Dataset, errerror)

GetSchemas returns schemas

func (*AzureTableConn)GetSchemata ¶added inv1.4.16

func (conn *AzureTableConn) GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (Schemata,error)

GetSchemata obtain full schemata info for a schema and/or table in current database

func (*AzureTableConn)GetTableColumns ¶added inv1.4.16

func (conn *AzureTableConn) GetTableColumns(table *Table, fields ...string) (columnsiop.Columns, errerror)

GetTableColumns returns columns for a table

func (*AzureTableConn)GetTables ¶added inv1.4.16

func (conn *AzureTableConn) GetTables(schemastring) (dataiop.Dataset, errerror)

GetTables returns tables

func (*AzureTableConn)Init ¶added inv1.4.16

func (conn *AzureTableConn) Init()error

Init initiates the object

func (*AzureTableConn)NewTransaction ¶added inv1.4.16

func (conn *AzureTableConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

func (*AzureTableConn)StreamRowsContext ¶added inv1.4.16

func (conn *AzureTableConn) StreamRowsContext(ctxcontext.Context, tableNamestring, Opts ...map[string]any) (ds *iop.Datastream, errerror)

func (*AzureTableConn)TableExists ¶added inv1.4.16

func (conn *AzureTableConn) TableExists(tableTable) (existsbool, errerror)

TableExists checks if a table exists

typeAzureTableResult ¶added inv1.4.16

type AzureTableResult struct {// contains filtered or unexported fields}

AzureTableResult implements sql.Result interface

func (*AzureTableResult)LastInsertId ¶added inv1.4.16

func (r *AzureTableResult) LastInsertId() (int64,error)

func (*AzureTableResult)RowsAffected ¶added inv1.4.16

func (r *AzureTableResult) RowsAffected() (int64,error)

typeBaseConn ¶

type BaseConn struct {ConnectionURLstringTypedbio.Type// the type of database for sqlx: postgres, mysql, sqliteDataiop.DatasetLog []string// contains filtered or unexported fields}

BaseConn is a database connection

func (*BaseConn)AddMissingColumns ¶added inv1.1.8

func (conn *BaseConn) AddMissingColumns(tableTable, newColsiop.Columns) (okbool, errerror)

func (*BaseConn)Base ¶

func (conn *BaseConn) Base() *BaseConn

BaseURL returns the base Conn

func (*BaseConn)BaseURL ¶

func (conn *BaseConn) BaseURL()string

BaseURL returns the base URL with default port

func (*BaseConn)Begin ¶

func (conn *BaseConn) Begin(options ...*sql.TxOptions) (errerror)

Begin starts a connection wide transaction

func (*BaseConn)BeginContext ¶

func (conn *BaseConn) BeginContext(ctxcontext.Context, options ...*sql.TxOptions) (errerror)

BeginContext starts a connection wide transaction

func (*BaseConn)BulkExportFlow ¶

func (conn *BaseConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

BulkExportFlow creates a dataflow from a sql query

func (*BaseConn)BulkExportFlowCSV ¶

func (conn *BaseConn) BulkExportFlowCSV(tableTable) (df *iop.Dataflow, errerror)

BulkExportFlowCSV creates a dataflow from a sql query, using CSVs

func (*BaseConn)BulkExportStream ¶

func (conn *BaseConn) BulkExportStream(tableTable) (ds *iop.Datastream, errerror)

BulkExportStream streams the rows in bulk

func (*BaseConn)BulkImportFlow ¶

func (conn *BaseConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow imports the streams rows in bulk concurrently using channels

func (*BaseConn)BulkImportStream ¶

func (conn *BaseConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream import the stream rows in bulk

func (*BaseConn)CastColumnForSelect ¶

func (conn *BaseConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column)string

CastColumnForSelect casts to the correct target column type

func (*BaseConn)CastColumnsForSelect ¶

func (conn *BaseConn) CastColumnsForSelect(srcColumnsiop.Columns, tgtColumnsiop.Columns) []string

CastColumnsForSelect cast the source columns into the target Column types

func (*BaseConn)Close ¶

func (conn *BaseConn) Close()error

Close closes the connection

func (*BaseConn)Commit ¶

func (conn *BaseConn) Commit() (errerror)

Commit commits a connection wide transaction

func (*BaseConn)CompareChecksums ¶

func (conn *BaseConn) CompareChecksums(tableNamestring, columnsiop.Columns) (errerror)

CompareChecksums compares the checksum values from the database sideto the checkum values from the StreamProcessor

func (*BaseConn)ConnString ¶

func (conn *BaseConn) ConnString()string

ConnString returns the connection string needed for connection

func (*BaseConn)Connect ¶

func (conn *BaseConn) Connect(timeOut ...int) (errerror)

Connect connects to the database

func (*BaseConn)Context ¶

func (conn *BaseConn) Context() *g.Context

Context returns the db context

func (*BaseConn)CreateTable ¶

func (conn *BaseConn) CreateTable(tableNamestring, colsiop.Columns, tableDDLstring) (errerror)

CreateTable creates a new table based on provided columns`tableName` should have 'schema.table' format

func (*BaseConn)CreateTemporaryTable ¶

func (conn *BaseConn) CreateTemporaryTable(tableNamestring, colsiop.Columns) (errerror)

CreateTemporaryTable creates a temp table based on provided columns

func (*BaseConn)CurrentDatabase ¶

func (conn *BaseConn) CurrentDatabase() (dbNamestring, errerror)

CurrentDatabase returns the name of the current database

func (*BaseConn)Db ¶

func (conn *BaseConn) Db() *sqlx.DB

Db returns the sqlx db object

func (*BaseConn)DbX ¶

func (conn *BaseConn) DbX() *DbX

DbX returns the DbX object

func (*BaseConn)DropTable ¶

func (conn *BaseConn) DropTable(tableNames ...string) (errerror)

DropTable drops given table.

func (*BaseConn)DropView ¶

func (conn *BaseConn) DropView(viewNames ...string) (errerror)

DropView drops given view.

func (*BaseConn)Exec ¶

func (conn *BaseConn) Exec(sqlstring, args ...interface{}) (resultsql.Result, errerror)

Exec runs a sql query, returns `error`

func (*BaseConn)ExecContext ¶

func (conn *BaseConn) ExecContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)

ExecContext runs a sql query with context, returns `error`

func (*BaseConn)ExecMulti ¶

func (conn *BaseConn) ExecMulti(sqls ...string) (resultsql.Result, errerror)

ExecMulti runs mutiple sql queries, returns `error`

func (*BaseConn)ExecMultiContext ¶

func (conn *BaseConn) ExecMultiContext(ctxcontext.Context, qs ...string) (resultsql.Result, errerror)

ExecMultiContext runs multiple sql queries with context, returns `error`

func (*BaseConn)GenerateDDL ¶

func (conn *BaseConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (string,error)

GenerateDDL genrate a DDL based on a dataset

func (*BaseConn)GenerateInsertStatement ¶

func (conn *BaseConn) GenerateInsertStatement(tableNamestring, colsiop.Columns, numRowsint)string

GenerateInsertStatement returns the proper INSERT statement

func (*BaseConn)GenerateMergeConfig ¶added inv1.4.20

func (conn *BaseConn) GenerateMergeConfig(srcTablestring, tgtTablestring, pkFields []string) (mcMergeConfig, errerror)

GenerateMergeConfig returns the merge config

func (*BaseConn)GenerateMergeExpressions ¶added inv1.4.20

func (conn *BaseConn) GenerateMergeExpressions(srcTablestring, tgtTablestring, pkFields []string) (upsertMap map[string]string, errerror)

GenerateMergeExpressions returns a map with needed expressions

func (*BaseConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *BaseConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL returns a sql for upsert

func (*BaseConn)GetAnalysis ¶

func (conn *BaseConn) GetAnalysis(analysisNamestring, values map[string]interface{}) (sqlstring, errerror)

GetAnalysis runs an analysis

func (*BaseConn)GetColumnStats ¶

func (conn *BaseConn) GetColumnStats(tableNamestring, fields ...string) (columnsiop.Columns, errerror)

GetColumnStats analyzes the table and returns the column statistics

func (*BaseConn)GetColumns ¶

func (conn *BaseConn) GetColumns(tableFNamestring, fields ...string) (columnsiop.Columns, errerror)

func (*BaseConn)GetColumnsFull ¶

func (conn *BaseConn) GetColumnsFull(tableFNamestring) (iop.Dataset,error)

func (*BaseConn)GetCount ¶

func (conn *BaseConn) GetCount(tableFNamestring) (int64,error)

GetCount returns count of records

func (*BaseConn)GetDDL ¶

func (conn *BaseConn) GetDDL(tableFNamestring) (string,error)

GetDDL returns DDL for given table.

func (*BaseConn)GetDatabases ¶

func (conn *BaseConn) GetDatabases() (iop.Dataset,error)

GetDatabases returns databases for given connection

func (*BaseConn)GetGormConn ¶

func (conn *BaseConn) GetGormConn(config *gorm.Config) (*gorm.DB,error)

GetGormConn returns the gorm db connection

func (*BaseConn)GetIndexes ¶

func (conn *BaseConn) GetIndexes(tableFNamestring) (iop.Dataset,error)

GetIndexes returns indexes for given table.

func (*BaseConn)GetMaxValue ¶added inv1.4.10

func (conn *BaseConn) GetMaxValue(tableTable, colNamestring) (valueany, maxColiop.Column, errerror)

GetMaxValue get the max value of a column

func (*BaseConn)GetNativeType ¶

func (conn *BaseConn) GetNativeType(coliop.Column) (nativeTypestring, errerror)

GetNativeType returns the native column type from generic

func (*BaseConn)GetObjects ¶

func (conn *BaseConn) GetObjects(schemastring, objectTypestring) (iop.Dataset,error)

GetObjects returns objects (tables or views) for given schema`objectType` can be either 'table', 'view' or 'all'

func (*BaseConn)GetPrimaryKeys ¶

func (conn *BaseConn) GetPrimaryKeys(tableFNamestring) (iop.Dataset,error)

GetPrimaryKeys returns primark keys for given table.

func (*BaseConn)GetProp ¶

func (conn *BaseConn) GetProp(key ...string)string

GetProp returns the value of a property

func (*BaseConn)GetSQLColumns ¶

func (conn *BaseConn) GetSQLColumns(tableTable) (columnsiop.Columns, errerror)

GetSQLColumns return columns from a sql query result

func (*BaseConn)GetSchemas ¶

func (conn *BaseConn) GetSchemas() (iop.Dataset,error)

GetSchemas returns schemas

func (*BaseConn)GetSchemata ¶

func (conn *BaseConn) GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (Schemata,error)

GetSchemata obtain full schemata info for a schema and/or table in current database

func (*BaseConn)GetTableColumns ¶

func (conn *BaseConn) GetTableColumns(table *Table, fields ...string) (columnsiop.Columns, errerror)

GetColumns returns columns for given table. `tableFName` shouldinclude schema and table, example: `schema1.table2`fields should be `column_name|data_type`

func (*BaseConn)GetTables ¶

func (conn *BaseConn) GetTables(schemastring) (iop.Dataset,error)

GetTables returns tables for given schema

func (*BaseConn)GetTablesAndViews ¶added inv1.2.19

func (conn *BaseConn) GetTablesAndViews(schemastring) (iop.Dataset,error)

GetTablesAndViews returns tables/views for given schema

func (*BaseConn)GetTemplateValue ¶

func (conn *BaseConn) GetTemplateValue(pathstring) (valuestring)

GetTemplateValue returns the value of the path

func (*BaseConn)GetType ¶

func (conn *BaseConn) GetType()dbio.Type

GetType returns the type db object

func (*BaseConn)GetURL ¶

func (conn *BaseConn) GetURL(newURL ...string)string

GetURL returns the processed URL

func (*BaseConn)GetViews ¶

func (conn *BaseConn) GetViews(schemastring) (iop.Dataset,error)

GetViews returns views for given schema

func (*BaseConn)Import ¶

func (conn *BaseConn) Import(dataiop.Dataset, tableNamestring)error

Import imports `data` into `tableName`

func (*BaseConn)Info ¶

func (conn *BaseConn) Info() (ciConnInfo)

Info returns connection information

func (*BaseConn)Init ¶

func (conn *BaseConn) Init() (errerror)

Init initiates the connection object & add default port if missing

func (*BaseConn)InsertBatchStream ¶

func (conn *BaseConn) InsertBatchStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertBatchStream inserts a stream into a table in batch

func (*BaseConn)InsertStream ¶

func (conn *BaseConn) InsertStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertStream inserts a stream into a table

func (*BaseConn)Kill ¶

func (conn *BaseConn) Kill()error

Kill kill the database connection

func (*BaseConn)LoadTemplates ¶

func (conn *BaseConn) LoadTemplates() (errerror)

LoadTemplates loads the appropriate yaml template

func (*BaseConn)LogSQL ¶added inv1.1.8

func (conn *BaseConn) LogSQL(querystring, args ...any)

LogSQL logs a query for debugging

func (*BaseConn)Merge ¶added inv1.4.20

func (conn *BaseConn) Merge(srcTablestring, tgtTablestring, primKeys []string) (rowAffCntint64, errerror)

Merge inserts / updates from a srcTable into a target table.Assuming the srcTable has some or all of the tgtTable fields with matching types

func (*BaseConn)MustExec ¶

func (conn *BaseConn) MustExec(sqlstring, args ...interface{}) (resultsql.Result)

MustExec execs the query using e and panics if there was an error.Any placeholder parameters are replaced with supplied args.

func (*BaseConn)NewTransaction ¶

func (conn *BaseConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (Transaction,error)

NewTransaction creates a new transaction

func (*BaseConn)OptimizeTable ¶

func (conn *BaseConn) OptimizeTable(table *Table, newColumnsiop.Columns, isTemp ...bool) (okbool, errerror)

func (*BaseConn)Prepare ¶

func (conn *BaseConn) Prepare(querystring) (stmt *sql.Stmt, errerror)

Prepare prepares the statement

func (*BaseConn)ProcessTemplate ¶

func (conn *BaseConn) ProcessTemplate(level, textstring, values map[string]interface{}) (sqlstring, errerror)

ProcessTemplate processes a template SQL text at a given level

func (*BaseConn)PropArr ¶

func (conn *BaseConn) PropArr() []string

PropArr returns an array of properties

func (*BaseConn)PropArrExclude ¶added inv1.2.14

func (conn *BaseConn) PropArrExclude(exclude ...string) []string

func (*BaseConn)Props ¶

func (conn *BaseConn) Props() map[string]string

Props returns a map properties

func (*BaseConn)Query ¶

func (conn *BaseConn) Query(sqlstring, options ...map[string]interface{}) (dataiop.Dataset, errerror)

Query runs a sql query, returns `result`, `error`

func (*BaseConn)QueryContext ¶

func (conn *BaseConn) QueryContext(ctxcontext.Context, sqlstring, options ...map[string]interface{}) (dataiop.Dataset, errerror)

QueryContext runs a sql query with ctx, returns `result`, `error`

func (*BaseConn)Quote ¶

func (conn *BaseConn) Quote(fieldstring)string

Quote adds quotes to the field name

func (*BaseConn)ReplaceProps ¶added inv1.2.16

func (conn *BaseConn) ReplaceProps(newProps map[string]string)

ReplaceProps used when reusing a connectionsince the provided props can change, this is usedto delete old original props and set new ones

func (*BaseConn)Rollback ¶

func (conn *BaseConn) Rollback() (errerror)

Rollback rolls back a connection wide transaction

func (*BaseConn)RunAnalysis ¶

func (conn *BaseConn) RunAnalysis(analysisNamestring, values map[string]interface{}) (dataiop.Dataset, errerror)

RunAnalysis runs an analysis

func (*BaseConn)Schemata ¶

func (conn *BaseConn) Schemata()Schemata

Schemata returns the Schemata object

func (*BaseConn)Self ¶

func (conn *BaseConn) Self()Connection

Self returns the respective connection InstanceThis is useful to refer back to a subclass methodfrom the superclass level. (Aka overloading)

func (*BaseConn)SetProp ¶

func (conn *BaseConn) SetProp(keystring, valstring)

SetProp sets the value of a property

func (*BaseConn)StreamRecords ¶

func (conn *BaseConn) StreamRecords(sqlstring) (<-chan map[string]interface{},error)

StreamRecords the records of a sql query, returns `result`, `error`

func (*BaseConn)StreamRows ¶

func (conn *BaseConn) StreamRows(sqlstring, options ...map[string]interface{}) (ds *iop.Datastream, errerror)

StreamRows the rows of a sql query, returns `result`, `error`

func (*BaseConn)StreamRowsContext ¶

func (conn *BaseConn) StreamRowsContext(ctxcontext.Context, querystring, options ...map[string]interface{}) (ds *iop.Datastream, errerror)

StreamRowsContext streams the rows of a sql query with context, returns `result`, `error`

func (*BaseConn)SubmitTemplate ¶added inv1.2.4

func (conn *BaseConn) SubmitTemplate(levelstring, templateMap map[string]string, namestring, values map[string]interface{}) (dataiop.Dataset, errerror)

func (*BaseConn)SwapTable ¶

func (conn *BaseConn) SwapTable(srcTablestring, tgtTablestring) (errerror)

SwapTable swaps two table

func (*BaseConn)TableExists ¶added inv1.4.10

func (conn *BaseConn) TableExists(tableTable) (existsbool, errerror)

TableExists returns true if the table exists

func (*BaseConn)Template ¶

func (conn *BaseConn) Template()dbio.Template

Template returns the Template object

func (*BaseConn)Tx ¶

func (conn *BaseConn) Tx()Transaction

Tx returns the current sqlx tx object

func (*BaseConn)Unquote ¶

func (conn *BaseConn) Unquote(fieldstring)string

Unquote removes quotes to the field name

func (*BaseConn)UseADBC ¶added inv1.5.2

func (conn *BaseConn) UseADBC()bool

UseADBC returns if connection should use ADBC

func (*BaseConn)ValidateColumnNames ¶

func (conn *BaseConn) ValidateColumnNames(tgtColsiop.Columns, colNames []string) (newColsiop.Columns, errerror)

ValidateColumnNames verifies that source fields are present in the target tableIt will return quoted field names as `newColNames`, the same length as `colNames`

typeBaseTransaction ¶

type BaseTransaction struct {Tx   *sqlx.TxConnConnection// contains filtered or unexported fields}

BaseTransaction is a database transaction

func (*BaseTransaction)Commit ¶

func (t *BaseTransaction) Commit() (errerror)

func (*BaseTransaction)Connection ¶

func (t *BaseTransaction) Connection()Connection

Connection return the connection

func (*BaseTransaction)Context ¶

func (t *BaseTransaction) Context() *g.Context

Commit commits connection wide transaction

func (*BaseTransaction)DisableTrigger ¶

func (t *BaseTransaction) DisableTrigger(tableName, triggerNamestring) (errerror)

DisableTrigger disables a trigger

func (*BaseTransaction)EnableTrigger ¶

func (t *BaseTransaction) EnableTrigger(tableName, triggerNamestring) (errerror)

EnableTrigger enables a trigger

func (*BaseTransaction)Exec ¶

func (t *BaseTransaction) Exec(sqlstring, args ...interface{}) (resultsql.Result, errerror)

Exec runs a sql query, returns `error`

func (*BaseTransaction)ExecContext ¶

func (t *BaseTransaction) ExecContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)

ExecContext runs a sql query with context, returns `error`

func (*BaseTransaction)ExecMultiContext ¶

func (t *BaseTransaction) ExecMultiContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)

ExecMultiContext runs multiple sql queries with context, returns `error`

func (*BaseTransaction)InsertBatchStream ¶

func (t *BaseTransaction) InsertBatchStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertBatchStream inserts a stream into a table in batch

func (*BaseTransaction)InsertStream ¶

func (t *BaseTransaction) InsertStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertStream inserts a stream into a table

func (*BaseTransaction)Merge ¶added inv1.4.20

func (t *BaseTransaction) Merge(sourceTable, targetTablestring, pkFields []string) (countuint64, errerror)

Merge does an upsert from source table into target table

func (*BaseTransaction)Prepare ¶

func (t *BaseTransaction) Prepare(querystring) (stmt *sql.Stmt, errerror)

Prepare prepares the statement

func (*BaseTransaction)QueryContext ¶

func (t *BaseTransaction) QueryContext(ctxcontext.Context, qstring, args ...interface{}) (result *sqlx.Rows, errerror)

QueryContext queries rows

func (*BaseTransaction)Rollback ¶

func (t *BaseTransaction) Rollback() (errerror)

Rollback rolls back connection wide transaction

func (*BaseTransaction)UpsertStream ¶

func (t *BaseTransaction) UpsertStream(tableFNamestring, ds *iop.Datastream, pk []string) (countuint64, errerror)

UpsertStream inserts a stream into a table in batch

typeBigQueryConn ¶

type BigQueryConn struct {BaseConnURLstringClient    *bigquery.ClientProjectIDstringDatasetIDstringLocationstringDatasets  []stringMuxsync.Mutex}

BigQueryConn is a Google Big Query connection

func (*BigQueryConn)BulkExportFlow ¶

func (conn *BigQueryConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

BulkExportFlow reads in bulk

func (*BigQueryConn)BulkImportFlow ¶

func (conn *BigQueryConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow inserts a flow of streams into a table.For redshift we need to create CSVs in GCS and then use the COPY command.

func (*BigQueryConn)BulkImportStream ¶

func (conn *BigQueryConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream demonstrates loading data into a BigQuery table using a file on the local filesystem.

func (*BigQueryConn)CastColumnForSelect ¶

func (conn *BigQueryConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*BigQueryConn)Close ¶

func (conn *BigQueryConn) Close()error

Close closes the connection

func (*BigQueryConn)Connect ¶

func (conn *BigQueryConn) Connect(timeOut ...int)error

Connect connects to the database

func (*BigQueryConn)CopyFromGCS ¶

func (conn *BigQueryConn) CopyFromGCS(gcsURIstring, tableTable, dsColumns []iop.Column, formatdbio.FileType)error

func (*BigQueryConn)CopyFromLocal ¶

func (conn *BigQueryConn) CopyFromLocal(localURIstring, tableTable, dsColumnsiop.Columns, formatdbio.FileType)error

CopyFromGCS into bigquery from google storage

func (*BigQueryConn)CopyToGCS ¶

func (conn *BigQueryConn) CopyToGCS(tableTable, gcsURIstring, formatdbio.FileType)error

func (*BigQueryConn)ExecContext ¶

func (conn *BigQueryConn) ExecContext(ctxcontext.Context, sqlstring, args ...interface{}) (resultsql.Result, errerror)

func (*BigQueryConn)ExportToGCS ¶

func (conn *BigQueryConn) ExportToGCS(sqlstring, gcsURIstring, formatdbio.FileType)error

CopyToGCS Copy table to gc storage

func (*BigQueryConn)GenerateDDL ¶added inv1.1.4

func (conn *BigQueryConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (sqlstring, errerror)

GenerateDDL generates a DDL based on a dataset

func (*BigQueryConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *BigQueryConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*BigQueryConn)GetDatabases ¶

func (conn *BigQueryConn) GetDatabases() (iop.Dataset,error)

GetDatabases returns databases

func (*BigQueryConn)GetSchemas ¶

func (conn *BigQueryConn) GetSchemas() (iop.Dataset,error)

GetSchemas returns schemas

func (*BigQueryConn)GetSchemata ¶

func (conn *BigQueryConn) GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (Schemata,error)

GetSchemata obtain full schemata info for a schema and/or table in current database

func (*BigQueryConn)Init ¶

func (conn *BigQueryConn) Init()error

Init initiates the object

func (*BigQueryConn)InsertBatchStream ¶

func (conn *BigQueryConn) InsertBatchStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertBatchStream inserts a stream into a table in batch

func (*BigQueryConn)InsertStream ¶

func (conn *BigQueryConn) InsertStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertStream demonstrates loading data into a BigQuery table using a file on the local filesystem.

func (*BigQueryConn)LoadFromReader ¶added inv1.4.24

func (conn *BigQueryConn) LoadFromReader(tableTable, readerio.Reader, dsColumns []iop.Column, formatdbio.FileType)error

LoadFromReader demonstrates loading data into a BigQuery table using a file on the local filesystem.https://cloud.google.com/bigquery/docs/batch-loading-data#loading_data_from_local_files

func (*BigQueryConn)NewTransaction ¶

func (conn *BigQueryConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

func (*BigQueryConn)StreamRowsContext ¶

func (conn *BigQueryConn) StreamRowsContext(ctxcontext.Context, sqlstring, options ...map[string]interface{}) (ds *iop.Datastream, errerror)

func (*BigQueryConn)Unload ¶

func (conn *BigQueryConn) Unload(tables ...Table) (gsPathstring, errerror)

Unload to Google Cloud Storage

typeBigTableAction ¶

type BigTableActionstring

const BTCreateColumnFamilyBigTableAction = "create_column_family"

const BTCreateTableBigTableAction = "create_table"

const BTDeleteTableBigTableAction = "delete_table"

const BTTableInfoBigTableAction = "table_info"

typeBigTableConn ¶

type BigTableConn struct {BaseConnURLstringClient     *bigtable.ClientProjectIDstringInstanceIDstringLocationstring}

BigTableConn is a Google Big Query connection

func (*BigTableConn)BulkExportFlow ¶

func (conn *BigTableConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

func (*BigTableConn)Close ¶

func (conn *BigTableConn) Close()error

Close closes the connection

func (*BigTableConn)Connect ¶

func (conn *BigTableConn) Connect(timeOut ...int)error

Connect connects to the database

func (*BigTableConn)ExecContext ¶

func (conn *BigTableConn) ExecContext(ctxcontext.Context, payloadstring, args ...interface{}) (resultsql.Result, errerror)

func (*BigTableConn)GetColumns ¶

func (conn *BigTableConn) GetColumns(tableFNamestring, fields ...string) (columnsiop.Columns, errerror)

func (*BigTableConn)GetColumnsFull ¶

func (conn *BigTableConn) GetColumnsFull(tableFNamestring) (iop.Dataset,error)

func (*BigTableConn)GetSQLColumns ¶

func (conn *BigTableConn) GetSQLColumns(tableTable) (columnsiop.Columns, errerror)

GetTables returns tables for given schema

func (*BigTableConn)GetSchemas ¶

func (conn *BigTableConn) GetSchemas() (iop.Dataset,error)

func (*BigTableConn)GetSchemata ¶

func (conn *BigTableConn) GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (schemataSchemata, errerror)

func (*BigTableConn)GetTables ¶

func (conn *BigTableConn) GetTables(schemastring) (dataiop.Dataset, errerror)

func (*BigTableConn)GetViews ¶

func (conn *BigTableConn) GetViews(schemastring) (dataiop.Dataset, errerror)

GetTables returns tables for given schema

func (*BigTableConn)Init ¶

func (conn *BigTableConn) Init()error

Init initiates the object

func (*BigTableConn)InsertBatchStream ¶

func (conn *BigTableConn) InsertBatchStream(tablestring, ds *iop.Datastream) (countuint64, errerror)

InsertBatchStream inserts a stream into a table in batch

func (*BigTableConn)NewTransaction ¶

func (conn *BigTableConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

func (*BigTableConn)StreamRowsContext ¶

func (conn *BigTableConn) StreamRowsContext(ctxcontext.Context, tablestring, options ...map[string]interface{}) (ds *iop.Datastream, errerror)

typeBigTableQuery ¶

type BigTableQuery struct {ActionBigTableAction `json:"action"`Tablestring         `json:"table"`ColumnFamilies []string       `json:"column_family"`}

typeBlankTransaction ¶

type BlankTransaction struct {ConnConnection// contains filtered or unexported fields}

func (*BlankTransaction)Commit ¶

func (t *BlankTransaction) Commit() (errerror)

func (*BlankTransaction)Connection ¶

func (t *BlankTransaction) Connection()Connection

func (*BlankTransaction)Context ¶

func (t *BlankTransaction) Context() *g.Context

func (*BlankTransaction)ExecContext ¶

func (t *BlankTransaction) ExecContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)

func (*BlankTransaction)ExecMultiContext ¶

func (t *BlankTransaction) ExecMultiContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)

func (*BlankTransaction)Prepare ¶

func (t *BlankTransaction) Prepare(querystring) (stmt *sql.Stmt, errerror)

func (*BlankTransaction)QueryContext ¶

func (t *BlankTransaction) QueryContext(ctxcontext.Context, qstring, args ...interface{}) (result *sqlx.Rows, errerror)

func (*BlankTransaction)Rollback ¶

func (t *BlankTransaction) Rollback() (errerror)

typeChunk ¶added inv1.4.14

type Chunk struct {RangeStartstring `json:"range_start,omitempty"`RangeEndstring `json:"range_end,omitempty"`Wherestring `json:"where,omitempty"`}

func (*Chunk)Range ¶added inv1.4.14

func (c *Chunk) Range()string

typeClickhouseConn ¶

type ClickhouseConn struct {BaseConnURLstring// contains filtered or unexported fields}

ClickhouseConn is a Clikchouse connection

func (*ClickhouseConn)BulkExportStream ¶added inv1.4.18

func (conn *ClickhouseConn) BulkExportStream(tableTable) (ds *iop.Datastream, errerror)

BulkExportStream streams the rows in bulk

func (*ClickhouseConn)BulkImportStream ¶

func (conn *ClickhouseConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream inserts a stream into a table

func (*ClickhouseConn)CastColumnForSelect ¶added inv1.5.1

func (conn *ClickhouseConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*ClickhouseConn)ConnString ¶

func (conn *ClickhouseConn) ConnString()string

func (*ClickhouseConn)Connect ¶added inv1.1.12

func (conn *ClickhouseConn) Connect(timeOut ...int) (errerror)

func (*ClickhouseConn)GenerateDDL ¶added inv1.1.4

func (conn *ClickhouseConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (ddlstring, errerror)

GenerateDDL generates a DDL based on a dataset

func (*ClickhouseConn)GenerateInsertStatement ¶

func (conn *ClickhouseConn) GenerateInsertStatement(tableNamestring, colsiop.Columns, numRowsint)string

GenerateInsertStatement returns the proper INSERT statement

func (*ClickhouseConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *ClickhouseConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*ClickhouseConn)GetNativeType ¶added inv1.2.15

func (conn *ClickhouseConn) GetNativeType(coliop.Column) (nativeTypestring, errerror)

func (*ClickhouseConn)Init ¶

func (conn *ClickhouseConn) Init()error

Init initiates the object

func (*ClickhouseConn)Merge ¶added inv1.4.21

func (conn *ClickhouseConn) Merge(srcTablestring, tgtTablestring, primKeys []string) (rowAffCntint64, errerror)

Merge inserts / updates from a srcTable into a target table.Assuming the srcTable has some or all of the tgtTable fields with matching types

func (*ClickhouseConn)NewTransaction ¶

func (conn *ClickhouseConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (Transaction,error)

NewTransaction creates a new transaction

func (*ClickhouseConn)Version ¶added inv1.4.21

func (conn *ClickhouseConn) Version()int

typeColumnType ¶

type ColumnType struct {NamestringDatabaseTypeNamestringFetchedColumn    *iop.ColumnLengthintPrecisionintScaleintNullableboolCT               *sql.ColumnTypeSourcedbool}

func (*ColumnType)IsSourced ¶added inv1.3.5

func (ct *ColumnType) IsSourced()bool

typeConnInfo ¶

type ConnInfo struct {HoststringPortintDatabasestringUserstringPasswordstringSchemastringWarehousestringRolestringURL       *net.URL}

typeConnection ¶

type Connection interface {Base() *BaseConnBaseURL()stringBegin(options ...*sql.TxOptions)errorBeginContext(ctxcontext.Context, options ...*sql.TxOptions)errorBulkExportFlow(tableTable) (*iop.Dataflow,error)BulkExportStream(tableTable) (*iop.Datastream,error)BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)CastColumnForSelect(srcColumniop.Column, tgtColumniop.Column)stringCastColumnsForSelect(srcColumnsiop.Columns, tgtColumnsiop.Columns) []stringClose()errorCommit()errorCompareChecksums(tableNamestring, columnsiop.Columns) (errerror)Connect(timeOut ...int)errorConnString()stringContext() *g.ContextCreateTable(tableNamestring, colsiop.Columns, tableDDLstring) (errerror)CreateTemporaryTable(tableNamestring, colsiop.Columns) (errerror)CurrentDatabase() (string,error)Db() *sqlx.DBDbX() *DbXDropTable(...string)errorDropView(...string)errorExec(sqlstring, args ...interface{}) (resultsql.Result, errerror)ExecContext(ctxcontext.Context, sqlstring, args ...interface{}) (resultsql.Result, errerror)ExecMulti(sqls ...string) (resultsql.Result, errerror)ExecMultiContext(ctxcontext.Context, sqls ...string) (resultsql.Result, errerror)GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (string,error)GenerateInsertStatement(tableNamestring, colsiop.Columns, numRowsint)stringGenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)GetAnalysis(string, map[string]interface{}) (string,error)GetColumns(tableFNamestring, fields ...string) (iop.Columns,error)GetColumnsFull(string) (iop.Dataset,error)GetColumnStats(tableNamestring, fields ...string) (columnsiop.Columns, errerror)GetCount(string) (int64,error)GetDatabases() (iop.Dataset,error)GetMaxValue(tTable, colNamestring) (valueany, coliop.Column, errerror)GetDDL(string) (string,error)GetGormConn(config *gorm.Config) (*gorm.DB,error)GetIndexes(string) (iop.Dataset,error)GetNativeType(coliop.Column) (nativeTypestring, errerror)GetPrimaryKeys(string) (iop.Dataset,error)GetProp(...string)stringGetSchemas() (iop.Dataset,error)GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (Schemata,error)GetSQLColumns(tableTable) (columnsiop.Columns, errerror)GetTableColumns(table *Table, fields ...string) (columnsiop.Columns, errerror)GetTablesAndViews(string) (iop.Dataset,error)GetTables(string) (iop.Dataset,error)GetTemplateValue(pathstring) (valuestring)GetType()dbio.TypeGetURL(newURL ...string)stringGetViews(string) (iop.Dataset,error)Info()ConnInfoInit()errorInsertBatchStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)InsertStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)Kill()errorLoadTemplates()errorMustExec(sqlstring, args ...interface{}) (resultsql.Result)NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (Transaction,error)OptimizeTable(table *Table, columnsiop.Columns, isTemp ...bool) (okbool, errerror)Prepare(querystring) (stmt *sql.Stmt, errerror)ProcessTemplate(level, textstring, values map[string]interface{}) (sqlstring, errerror)Props() map[string]stringPropsArr() []stringQuery(sqlstring, options ...map[string]interface{}) (iop.Dataset,error)QueryContext(ctxcontext.Context, sqlstring, options ...map[string]interface{}) (iop.Dataset,error)Quote(fieldstring)stringRenameTable(tablestring, newTablestring) (errerror)Rollback()errorRunAnalysis(string, map[string]interface{}) (iop.Dataset,error)Schemata()SchemataSelf()ConnectionSetProp(string,string)StreamRecords(sqlstring) (<-chan map[string]interface{},error)StreamRows(sqlstring, options ...map[string]interface{}) (*iop.Datastream,error)StreamRowsContext(ctxcontext.Context, sqlstring, options ...map[string]interface{}) (ds *iop.Datastream, errerror)SubmitTemplate(levelstring, templateMap map[string]string, namestring, values map[string]interface{}) (dataiop.Dataset, errerror)SwapTable(srcTablestring, tgtTablestring) (errerror)TableExists(tableTable) (existsbool, errerror)Template()dbio.TemplateTx()TransactionUnquote(string)stringMerge(srcTablestring, tgtTablestring, pkFields []string) (rowAffCntint64, errerror)ValidateColumnNames(tgtColsiop.Columns, colNames []string) (newColsiop.Columns, errerror)AddMissingColumns(tableTable, newColsiop.Columns) (okbool, errerror)UseADBC()bool}

Connection is the Base interface for Connections

funcClone ¶

func Clone(connConnection) (newConnConnection, errerror)

funcNewAdbcConn ¶added inv1.5.2

func NewAdbcConn(parentConnConnection) (adbcConnConnection, errerror)

NewAdbcConn creates a new ADBC conn from a parent connconstructs the connection string with complete URIs/paths for each database type

funcNewConn ¶

func NewConn(URLstring, props ...string) (Connection,error)

NewConn return the most proper connection for a given database

funcNewConnContext ¶

func NewConnContext(ctxcontext.Context, URLstring, props ...string) (Connection,error)

NewConnContext return the most proper connection for a given database with contextprops are provided as `"Prop1=Value1", "Prop2=Value2", ...`

typeD1Conn ¶added inv1.3.5

type D1Conn struct {SQLiteConnURLstringAccountIDstringDatabasestringUUIDstringAPITokenstring// contains filtered or unexported fields}

D1Conn is a Cloudflare SQLite connection

func (*D1Conn)BulkImportStream ¶added inv1.3.5

func (conn *D1Conn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

func (*D1Conn)Connect ¶added inv1.3.5

func (conn *D1Conn) Connect(timeOut ...int) (errerror)

Connect connects to the database

func (*D1Conn)ExecContext ¶added inv1.3.5

func (conn *D1Conn) ExecContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)

ExecContext runs a sql query with context, returns `error`

func (*D1Conn)GenerateMergeSQL ¶added inv1.4.20

func (conn *D1Conn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*D1Conn)GetDatabases ¶added inv1.4.6

func (conn *D1Conn) GetDatabases() (dataiop.Dataset, errerror)

GetDatabases returns databases for given connection

func (*D1Conn)GetSchemata ¶added inv1.3.5

func (conn *D1Conn) GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (Schemata,error)

GetSchemata obtain full schemata info for a schema and/or table in current database

func (*D1Conn)Init ¶added inv1.3.5

func (conn *D1Conn) Init()error

Init initiates the object

func (*D1Conn)InsertBatchStream ¶added inv1.3.5

func (conn *D1Conn) InsertBatchStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

func (*D1Conn)StreamRowsContext ¶added inv1.3.5

func (conn *D1Conn) StreamRowsContext(ctxcontext.Context, querystring, options ...map[string]interface{}) (ds *iop.Datastream, errerror)

typeDataAnalyzer ¶

type DataAnalyzer struct {ConnConnectionSchemataSchemataColumnMap   map[string]iop.ColumnRelationMap map[string]map[string]map[string]Relation// table > column A > column B > relationOptionsDataAnalyzerOptions}

funcNewDataAnalyzer ¶

func NewDataAnalyzer(connConnection, optsDataAnalyzerOptions) (da *DataAnalyzer, errerror)

func (*DataAnalyzer)AnalyzeColumns ¶

func (da *DataAnalyzer) AnalyzeColumns(sampleSizeint, includeViewsbool) (errerror)

func (*DataAnalyzer)GetManyToMany ¶

func (da *DataAnalyzer) GetManyToMany(nonUniqueColsiop.Columns, asStringbool) (errerror)

func (*DataAnalyzer)GetOneToMany ¶

func (da *DataAnalyzer) GetOneToMany(uniqueCols, nonUniqueColsiop.Columns, asStringbool) (errerror)

func (*DataAnalyzer)GetOneToOne ¶

func (da *DataAnalyzer) GetOneToOne(uniqueColsiop.Columns, asStringbool) (errerror)

func (*DataAnalyzer)GetSchemata ¶

func (da *DataAnalyzer) GetSchemata(forcebool) (errerror)

func (*DataAnalyzer)ProcessRelations ¶

func (da *DataAnalyzer) ProcessRelations() (errerror)

func (*DataAnalyzer)ProcessRelationsInteger ¶

func (da *DataAnalyzer) ProcessRelationsInteger() (errerror)

func (*DataAnalyzer)ProcessRelationsString ¶

func (da *DataAnalyzer) ProcessRelationsString() (errerror)

func (*DataAnalyzer)WriteRelationsYaml ¶

func (da *DataAnalyzer) WriteRelationsYaml(pathstring) (errerror)

typeDataAnalyzerOptions ¶

type DataAnalyzerOptions struct {DbNamestringSchemaNames []string}

typeDatabase ¶

type Database struct {Namestring            `json:"name"`Schemas map[string]Schema `json:"schemas"`}

Database represents a schemata database

func (*Database)Columns ¶

func (db *Database) Columns() map[string]iop.Column

func (*Database)Tables ¶

func (db *Database) Tables() map[string]Table

typeDatabricksConn ¶added inv1.4.11

type DatabricksConn struct {BaseConnURLstringCatalogstringSchemastringWarehousestringCopyMethodstring}

DatabricksConn is a Databricks connection

func (*DatabricksConn)BulkExportFlow ¶added inv1.4.11

func (conn *DatabricksConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

BulkExportFlow reads in bulk

func (*DatabricksConn)BulkImportFlow ¶added inv1.4.11

func (conn *DatabricksConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow inserts a flow of streams into a table

func (*DatabricksConn)BulkImportStream ¶added inv1.4.11

func (conn *DatabricksConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream inserts a stream into a table

func (*DatabricksConn)CastColumnForSelect ¶added inv1.4.11

func (conn *DatabricksConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*DatabricksConn)Connect ¶added inv1.4.11

func (conn *DatabricksConn) Connect(timeOut ...int)error

Connect connects to the database

func (*DatabricksConn)CopyFromS3 ¶added inv1.4.11

func (conn *DatabricksConn) CopyFromS3(tableFName, s3Path, fileFormatstring) (errerror)

CopyFromS3 uses the Databricks COPY INTO command from AWS S3

func (*DatabricksConn)CopyFromVolume ¶added inv1.4.11

func (conn *DatabricksConn) CopyFromVolume(tableFName, volumePathstring, fileFormatdbio.FileType, columnsiop.Columns)error

CopyFromVolume uses the Databricks COPY INTO command from volumes

func (*DatabricksConn)CopyToS3 ¶added inv1.4.11

func (conn *DatabricksConn) CopyToS3(tableTable) (s3Pathstring, errerror)

CopyToS3 exports a query to an S3 location using CREATE EXTERNAL TABLE

func (*DatabricksConn)CopyViaS3 ¶added inv1.4.11

func (conn *DatabricksConn) CopyViaS3(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

CopyViaS3 uses the Databricks COPY INTO command from AWS S3

func (*DatabricksConn)CopyViaVolume ¶added inv1.4.11

func (conn *DatabricksConn) CopyViaVolume(tableTable, df *iop.Dataflow) (countuint64, errerror)

CopyViaVolume uses Databricks volumes for bulk import, similar to Snowflake's CopyViaStage

func (*DatabricksConn)GenerateDDL ¶added inv1.4.11

func (conn *DatabricksConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (sqlstring, errerror)

GenerateDDL generates a DDL based on a dataset

func (*DatabricksConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *DatabricksConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*DatabricksConn)GetColumnsFull ¶added inv1.4.11

func (conn *DatabricksConn) GetColumnsFull(tableFNamestring) (dataiop.Dataset, errerror)

GetColumnsFull returns full column information for a table

func (*DatabricksConn)GetSchemata ¶added inv1.4.24

func (conn *DatabricksConn) GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (Schemata,error)

func (*DatabricksConn)GetURL ¶added inv1.4.11

func (conn *DatabricksConn) GetURL(newURL ...string)string

GetURL returns the URL

func (*DatabricksConn)Init ¶added inv1.4.11

func (conn *DatabricksConn) Init()error

Init initiates the object

func (*DatabricksConn)NewTransaction ¶added inv1.4.11

func (conn *DatabricksConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

func (*DatabricksConn)UnloadViaVolume ¶added inv1.4.11

func (conn *DatabricksConn) UnloadViaVolume(tables ...Table) (filePathstring, unloadedFilesint, errerror)

UnloadViaVolume exports data to a Databricks volume, similar to Snowflake's UnloadViaStage

func (*DatabricksConn)VolumeDelete ¶added inv1.4.11

func (conn *DatabricksConn) VolumeDelete(volumePaths ...string) (errerror)

VolumeDelete delete files in a Databricks volume path

func (*DatabricksConn)VolumeGET ¶added inv1.4.11

func (conn *DatabricksConn) VolumeGET(volumePath, folderPath, localFilePathstring)error

VolumeGET downloads a file from a Databricks volume to local filesystem

func (*DatabricksConn)VolumeList ¶added inv1.4.11

func (conn *DatabricksConn) VolumeList(volumePathstring) (dataiop.Dataset, errerror)

VolumeList lists files in a Databricks volume path

func (*DatabricksConn)VolumePUT ¶added inv1.4.11

func (conn *DatabricksConn) VolumePUT(folderPath, localFilePath, volumePathstring)error

VolumePUT uploads a local file to a Databricks volume using SQL commands

typeDbX ¶

type DbX struct {// contains filtered or unexported fields}

DbX is db express

func (*DbX)Delete ¶

func (x *DbX) Delete(o interface{}) (cntint, errerror)

Delete deletes from object or slice

func (*DbX)Get ¶

func (x *DbX) Get(o interface{}, fields ...string) (errerror)

Get retrieves object

func (*DbX)Insert ¶

func (x *DbX) Insert(o interface{}, fields ...string) (errerror)

Insert inserts object or slice

func (*DbX)Merge ¶added inv1.4.20

func (x *DbX) Merge(o interface{}, fields ...string) (cntint, errerror)

Merge upserts from object or slice

func (*DbX)Select ¶

func (x *DbX) Select(o interface{}, fields ...string) (errerror)

Select retrieves objects

func (*DbX)TableName ¶

func (x *DbX) TableName(o interface{}) (namestring)

TableName returns the table name of object or slice

func (*DbX)Update ¶

func (x *DbX) Update(o interface{}, fields ...string) (cntint, errerror)

Update updates from object or slice

func (*DbX)Where ¶

func (x *DbX) Where(where ...interface{}) *DbX

Where adds a where clause

typeDuckDbConn ¶

type DuckDbConn struct {BaseConnURLstring// contains filtered or unexported fields}

DuckDbConn is a Duck DB connection

func (*DuckDbConn)BulkImportFlow ¶added inv1.2.19

func (conn *DuckDbConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

func (*DuckDbConn)CastColumnForSelect ¶added inv1.2.10

func (conn *DuckDbConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*DuckDbConn)Close ¶

func (conn *DuckDbConn) Close() (errerror)

func (*DuckDbConn)Connect ¶

func (conn *DuckDbConn) Connect(timeOut ...int) (errerror)

func (*DuckDbConn)DuckDb ¶added inv1.2.25

func (conn *DuckDbConn) DuckDb() *iop.DuckDb

DuckDb returns the DuckDb instance

func (*DuckDbConn)ExecContext ¶

func (conn *DuckDbConn) ExecContext(ctxcontext.Context, sqlstring, args ...interface{}) (resultsql.Result, errerror)

func (*DuckDbConn)ExecMultiContext ¶

func (conn *DuckDbConn) ExecMultiContext(ctxcontext.Context, sqls ...string) (resultsql.Result, errerror)

ExecContext runs a sql query with context, returns `error`

func (*DuckDbConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *DuckDbConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*DuckDbConn)GetURL ¶

func (conn *DuckDbConn) GetURL(newURL ...string)string

GetURL returns the processed URL

func (*DuckDbConn)Init ¶

func (conn *DuckDbConn) Init()error

Init initiates the object

func (*DuckDbConn)InsertBatchStream ¶

func (conn *DuckDbConn) InsertBatchStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertBatchStream inserts a stream into a table in batch

func (*DuckDbConn)InsertStream ¶

func (conn *DuckDbConn) InsertStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertStream demonstrates loading data into a BigQuery table using a file on the local filesystem.

func (*DuckDbConn)StreamRowsContext ¶

func (conn *DuckDbConn) StreamRowsContext(ctxcontext.Context, sqlstring, options ...map[string]interface{}) (ds *iop.Datastream, errerror)

typeDuckLakeConn ¶added inv1.4.10

type DuckLakeConn struct {DuckDbConn// Catalog database configurationCatalogTypestring// duckdb, sqlite, postgres, mysqlCatalogSchemastring// schema to use in postgres or mysqlCatalogConnStrstring// connection string for catalog databaseDataPathstring// path to data files (local or cloud storage)Databasestring// the database name to attachedEncryptedbool// whether data is written using Parquet encryptionInliningLimitint// where to enable data inlining when attaching}

DuckLakeConn is a Ducklake connection

func (*DuckLakeConn)Close ¶added inv1.4.10

func (conn *DuckLakeConn) Close()error

Close closes the DuckLake connection

func (*DuckLakeConn)Connect ¶added inv1.4.10

func (conn *DuckLakeConn) Connect(timeOut ...int) (errerror)

Connect establishes the DuckLake connection

func (*DuckLakeConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *DuckLakeConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL for DuckLake

func (*DuckLakeConn)GetURL ¶added inv1.4.10

func (conn *DuckLakeConn) GetURL(newURL ...string)string

GetURL returns the processed URL

func (*DuckLakeConn)Init ¶added inv1.4.10

func (conn *DuckLakeConn) Init()error

Init initiates the object

func (*DuckLakeConn)SubmitTemplate ¶added inv1.4.10

func (conn *DuckLakeConn) SubmitTemplate(levelstring, templateMap map[string]string, namestring, values map[string]any) (dataiop.Dataset, errerror)

typeElasticsearchConn ¶added inv1.3.5

type ElasticsearchConn struct {BaseConnURLstringClient *elasticsearch.Client}

ElasticsearchConn is a elasticsearch connection

func (*ElasticsearchConn)BulkExportFlow ¶added inv1.3.5

func (conn *ElasticsearchConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

func (*ElasticsearchConn)Close ¶added inv1.3.5

func (conn *ElasticsearchConn) Close()error

func (*ElasticsearchConn)Connect ¶added inv1.3.5

func (conn *ElasticsearchConn) Connect(timeOut ...int)error

Connect connects to the database

func (*ElasticsearchConn)ExecContext ¶added inv1.3.5

func (conn *ElasticsearchConn) ExecContext(ctxcontext.Context, sqlstring, args ...interface{}) (resultsql.Result, errerror)

func (*ElasticsearchConn)GetSchemas ¶added inv1.3.5

func (conn *ElasticsearchConn) GetSchemas() (dataiop.Dataset, errerror)

GetSchemas returns schemas

func (*ElasticsearchConn)GetSchemata ¶added inv1.3.5

func (conn *ElasticsearchConn) GetSchemata(levelSchemataLevel, schemastring, tables ...string) (schemataSchemata, errerror)

GetSchemata returns the database schemata

func (*ElasticsearchConn)GetTableColumns ¶added inv1.3.5

func (conn *ElasticsearchConn) GetTableColumns(table *Table, fields ...string) (columnsiop.Columns, errerror)

GetTableColumns returns columns for a table

func (*ElasticsearchConn)GetTables ¶added inv1.3.5

func (conn *ElasticsearchConn) GetTables(schemastring) (dataiop.Dataset, errerror)

GetTables returns tables

func (*ElasticsearchConn)Init ¶added inv1.3.5

func (conn *ElasticsearchConn) Init()error

Init initiates the object

func (*ElasticsearchConn)NewTransaction ¶added inv1.3.5

func (conn *ElasticsearchConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

func (*ElasticsearchConn)StreamRowsContext ¶added inv1.3.5

func (conn *ElasticsearchConn) StreamRowsContext(ctxcontext.Context, tableNamestring, Opts ...map[string]any) (ds *iop.Datastream, errerror)

typeExasolConn ¶added inv1.4.16

type ExasolConn struct {BaseConnURLstring}

ExasolConn is a Exasol connection

func (*ExasolConn)BulkImportFlow ¶added inv1.4.16

func (conn *ExasolConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow performs bulk import for Exasol using temporary CSV files

func (*ExasolConn)BulkImportStream ¶added inv1.4.16

func (conn *ExasolConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream performs bulk import for Exasol

func (*ExasolConn)Connect ¶added inv1.4.16

func (conn *ExasolConn) Connect(timeOut ...int)error

Connect connects to the database

func (*ExasolConn)GenerateDDL ¶added inv1.4.16

func (conn *ExasolConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (string,error)

GenerateDDL generates DDL for Exasol

func (*ExasolConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *ExasolConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL for Exasol

func (*ExasolConn)GetURL ¶added inv1.4.16

func (conn *ExasolConn) GetURL(newURL ...string)string

GetURL returns the processed URLThis method uses the official Exasol driver's configuration builder (exasol.NewConfig)to properly handle special characters in passwords and other connection parameters.This ensures passwords with backslashes, semicolons, etc. are correctly escaped.

func (*ExasolConn)Init ¶added inv1.4.16

func (conn *ExasolConn) Init()error

Init initiates the connection

func (*ExasolConn)NewTransaction ¶added inv1.4.16

func (conn *ExasolConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

typeIcebergConn ¶added inv1.4.10

type IcebergConn struct {BaseConnURLstringCatalogTypedbio.IcebergCatalogTypeCatalogSQLConnConnectionCatalogcatalog.CatalogWarehousestring// contains filtered or unexported fields}

IcebergConn is an Iceberg connection

func (*IcebergConn)BulkExportStream ¶added inv1.4.10

func (conn *IcebergConn) BulkExportStream(tableTable) (ds *iop.Datastream, errerror)

BulkExportStream reads table data in bulk

func (*IcebergConn)BulkImportStream ¶added inv1.4.10

func (conn *IcebergConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream inserts a stream into a table using Arrow format.This method converts the incoming datastream to Apache Arrow format and usesthe iceberg-go table.AppendTable API to write the data.

func (*IcebergConn)CastColumnForSelect ¶added inv1.4.10

func (conn *IcebergConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*IcebergConn)Close ¶added inv1.4.10

func (conn *IcebergConn) Close()error

Close closes the connection

func (*IcebergConn)Connect ¶added inv1.4.10

func (conn *IcebergConn) Connect(timeOut ...int) (errerror)

Connect connects to the Iceberg catalog

func (*IcebergConn)CreateNamespaceIfNotExists ¶added inv1.4.10

func (conn *IcebergConn) CreateNamespaceIfNotExists(schemastring) (errerror)

func (*IcebergConn)CreateTable ¶added inv1.4.10

func (conn *IcebergConn) CreateTable(tableNamestring, colsiop.Columns, tableDDLstring) (errerror)

func (*IcebergConn)DropTable ¶added inv1.4.10

func (conn *IcebergConn) DropTable(tableNames ...string) (errerror)

DropTable drops given table.

func (*IcebergConn)ExecContext ¶added inv1.4.10

func (conn *IcebergConn) ExecContext(ctxcontext.Context, sqlstring, args ...interface{}) (resultsql.Result, errerror)

ExecContext executes a write operation

func (*IcebergConn)GenerateDDL ¶added inv1.4.10

func (conn *IcebergConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (sqlstring, errerror)

GenerateDDL generates a DDL based on a dataset

func (*IcebergConn)GetColumns ¶added inv1.4.10

func (conn *IcebergConn) GetColumns(tableFNamestring, fields ...string) (columnsiop.Columns, errerror)

GetColumns returns the columns for a given table

func (*IcebergConn)GetCount ¶added inv1.4.10

func (conn *IcebergConn) GetCount(tableFNamestring) (countint64, errerror)

GetCount returns -1 to skip validation

func (*IcebergConn)GetDataFiles ¶added inv1.4.10

func (conn *IcebergConn) GetDataFiles(tTable) (dataFiles []iceberg.DataFile, errerror)

func (*IcebergConn)GetDatabases ¶added inv1.4.10

func (conn *IcebergConn) GetDatabases() (dataiop.Dataset, errerror)

GetDatabases returns databases (catalogs) for Iceberg connection

func (*IcebergConn)GetMaxValue ¶added inv1.4.10

func (conn *IcebergConn) GetMaxValue(tTable, colNamestring) (valueany, maxColiop.Column, errerror)

GetMaxValue gets the maximum value of the given column

func (*IcebergConn)GetSchemas ¶added inv1.4.10

func (conn *IcebergConn) GetSchemas() (dataiop.Dataset, errerror)

GetSchemas returns the list of namespaces (schemas)

func (*IcebergConn)GetSchemata ¶added inv1.4.10

func (conn *IcebergConn) GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (Schemata,error)

GetSchemata obtain full schemata info for a schema and/or table

func (*IcebergConn)GetTableColumns ¶added inv1.4.10

func (conn *IcebergConn) GetTableColumns(table *Table, fields ...string) (columnsiop.Columns, errerror)

func (*IcebergConn)GetTables ¶added inv1.4.10

func (conn *IcebergConn) GetTables(schemastring) (dataiop.Dataset, errerror)

GetTables returns tables for given schema

func (*IcebergConn)GetTablesAndViews ¶added inv1.4.10

func (conn *IcebergConn) GetTablesAndViews(schemastring) (dataiop.Dataset, errerror)

GetTablesAndViews returns tables and views for given schema

func (*IcebergConn)GetViews ¶added inv1.4.10

func (conn *IcebergConn) GetViews(schemastring) (dataiop.Dataset, errerror)

GetViews returns views for given schema

func (*IcebergConn)Init ¶added inv1.4.10

func (conn *IcebergConn) Init()error

Init initiates the object

func (*IcebergConn)InsertBatchStream ¶added inv1.4.10

func (conn *IcebergConn) InsertBatchStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertBatchStream inserts a stream into a table in batch

func (*IcebergConn)InsertStream ¶added inv1.4.10

func (conn *IcebergConn) InsertStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertStream inserts data into a table

func (*IcebergConn)NewTransaction ¶added inv1.4.10

func (conn *IcebergConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

func (*IcebergConn)StreamRowsContext ¶added inv1.4.10

func (conn *IcebergConn) StreamRowsContext(ctxcontext.Context, sqlstring, options ...map[string]interface{}) (ds *iop.Datastream, errerror)

StreamRowsContext streams the rows of a table or query

func (*IcebergConn)SwapTable ¶added inv1.4.10

func (conn *IcebergConn) SwapTable(srcTablestring, tgtTablestring) (errerror)

SwapTable swaps two tables by renaming them2025-06-09 => doesn't work, blank error

func (*IcebergConn)TableExists ¶added inv1.4.10

func (conn *IcebergConn) TableExists(tTable) (existsbool, errerror)

typeIsolationLevel ¶added inv1.4.11

type IsolationLevelstring

const (IsolationLevelDefaultIsolationLevel = "default"IsolationLevelReadUncommittedIsolationLevel = "read_uncommitted"IsolationLevelReadCommittedIsolationLevel = "read_committed"IsolationLevelWriteCommittedIsolationLevel = "write_committed"IsolationLevelRepeatableReadIsolationLevel = "repeatable_read"IsolationLevelSnapshotIsolationLevel = "snapshot"IsolationLevelSerializableIsolationLevel = "serializable"IsolationLevelLinearizableIsolationLevel = "linearizable")

func (IsolationLevel)AsSqlIsolationLevel ¶added inv1.4.11

func (ilIsolationLevel) AsSqlIsolationLevel()sql.IsolationLevel

typeManualTransaction ¶

type ManualTransaction struct {ConnConnection// contains filtered or unexported fields}

func (*ManualTransaction)Commit ¶

func (t *ManualTransaction) Commit() (errerror)

func (*ManualTransaction)Context ¶

func (t *ManualTransaction) Context() *g.Context

func (*ManualTransaction)ExecContext ¶

func (t *ManualTransaction) ExecContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)

func (*ManualTransaction)ExecMultiContext ¶

func (t *ManualTransaction) ExecMultiContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)

func (*ManualTransaction)Prepare ¶

func (t *ManualTransaction) Prepare(querystring) (stmt *sql.Stmt, errerror)

func (*ManualTransaction)QueryContext ¶

func (t *ManualTransaction) QueryContext(ctxcontext.Context, qstring, args ...interface{}) (result *sqlx.Rows, errerror)

func (*ManualTransaction)Rollback ¶

func (t *ManualTransaction) Rollback() (errerror)

typeMergeConfig ¶added inv1.4.20

type MergeConfig struct {StrategyMergeStrategyTemplatestringMap      map[string]string}

typeMergeStrategy ¶added inv1.4.20

type MergeStrategystring

MergeStrategy is for incremental loading

const (MergeStrategyNoneMergeStrategy = ""MergeStrategyInsertMergeStrategy = "insert"MergeStrategyUpdateMergeStrategy = "update"MergeStrategyUpdateInsertMergeStrategy = "update_insert"MergeStrategyDeleteInsertMergeStrategy = "delete_insert"MergeStrategyHistoryInsertMergeStrategy = "history_insert")

typeModelDbX ¶

type ModelDbX struct {Ptr          interface{} `json:"-"`RowsAffectedint         `json:"-"`// contains filtered or unexported fields}

ModelDbX is the base for any SQL model

func (*ModelDbX)Bind ¶

func (m *ModelDbX) Bind(bindFunc func(p interface{})error, objPtr interface{}) (errerror)

Bind extracts values from provided echo context

func (*ModelDbX)Delete ¶

func (m *ModelDbX) Delete(db *sqlx.DB) (errerror)

Delete deletes a record

func (*ModelDbX)Fields ¶

func (m *ModelDbX) Fields() (fields []string)

Fields returns the model fields

func (*ModelDbX)Get ¶

func (m *ModelDbX) Get(db *sqlx.DB, fields ...string) (errerror)

Get get the first record

func (*ModelDbX)Insert ¶

func (m *ModelDbX) Insert(db *sqlx.DB, fields ...string) (errerror)

Insert inserts one records

func (*ModelDbX)Rec ¶

func (m *ModelDbX) Rec() map[string]interface{}

Rec returns the record

func (*ModelDbX)Select ¶

func (m *ModelDbX) Select(db *sqlx.DB, objPtr interface{}, fields ...string) (errerror)

Select returns multiple records

func (*ModelDbX)TableName ¶

func (m *ModelDbX) TableName(objPtr interface{})string

TableName returns the table name of the underlying pointer

func (*ModelDbX)Update ¶

func (m *ModelDbX) Update(db *sqlx.DB, fields ...string) (errerror)

Insert inserts one records

func (*ModelDbX)Values ¶

func (m *ModelDbX) Values(fields []string) (values []interface{}, errerror)

Where adds a where clause

func (*ModelDbX)Where ¶

func (m *ModelDbX) Where(where ...interface{}) *ModelDbX

Where adds a where clause

typeMongoDBConn ¶added inv1.1.14

type MongoDBConn struct {BaseConnURLstringClient *mongo.Client}

MongoDBConn is a Mongo connection

func (*MongoDBConn)BulkExportFlow ¶added inv1.1.14

func (conn *MongoDBConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

func (*MongoDBConn)Close ¶added inv1.1.14

func (conn *MongoDBConn) Close()error

func (*MongoDBConn)Connect ¶added inv1.1.14

func (conn *MongoDBConn) Connect(timeOut ...int)error

Connect connects to the database

func (*MongoDBConn)ExecContext ¶added inv1.1.14

func (conn *MongoDBConn) ExecContext(ctxcontext.Context, sqlstring, args ...any) (resultsql.Result, errerror)

func (*MongoDBConn)GetSchemas ¶added inv1.1.14

func (conn *MongoDBConn) GetSchemas() (dataiop.Dataset, errerror)

GetSchemas returns schemas

func (*MongoDBConn)GetSchemata ¶added inv1.1.15

func (conn *MongoDBConn) GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (Schemata,error)

GetSchemata obtain full schemata info for a schema and/or table in current database

func (*MongoDBConn)GetTableColumns ¶added inv1.1.14

func (conn *MongoDBConn) GetTableColumns(table *Table, fields ...string) (columnsiop.Columns, errerror)

NewTransaction creates a new transaction

func (*MongoDBConn)GetTables ¶added inv1.1.14

func (conn *MongoDBConn) GetTables(schemastring) (dataiop.Dataset, errerror)

GetSchemas returns schemas

func (*MongoDBConn)Init ¶added inv1.1.14

func (conn *MongoDBConn) Init()error

Init initiates the object

func (*MongoDBConn)NewTransaction ¶added inv1.1.14

func (conn *MongoDBConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

func (*MongoDBConn)StreamRowsContext ¶added inv1.1.14

func (conn *MongoDBConn) StreamRowsContext(ctxcontext.Context, collectionNamestring, Opts ...map[string]any) (ds *iop.Datastream, errerror)

typeMsFabricConn ¶added inv1.4.24

type MsFabricConn struct {MsSQLServerConnURLstring// contains filtered or unexported fields}

MsFabricConn is a Microsoft Fabric connection

func (*MsFabricConn)AddMissingColumns ¶added inv1.4.24

func (conn *MsFabricConn) AddMissingColumns(tableTable, newColsiop.Columns) (okbool, errerror)

func (*MsFabricConn)BulkExportFlow ¶added inv1.4.24

func (conn *MsFabricConn) BulkExportFlow(tableTable) (*iop.Dataflow,error)

BulkExportFlow exports data using cursor (OneLake export not yet supported)

func (*MsFabricConn)BulkImportFlow ¶added inv1.4.24

func (conn *MsFabricConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow bulk imports using OneLake staging

func (*MsFabricConn)BulkImportStream ¶added inv1.4.24

func (conn *MsFabricConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream bulk imports a stream using OneLake staging

func (*MsFabricConn)CastColumnForSelect ¶added inv1.4.24

func (conn *MsFabricConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*MsFabricConn)Connect ¶added inv1.4.24

func (conn *MsFabricConn) Connect(timeOut ...int) (errerror)

func (*MsFabricConn)CopyFromOneLake ¶added inv1.4.24

func (conn *MsFabricConn) CopyFromOneLake(tableFName, oneLakePathstring, columnsiop.Columns, fileFormatdbio.FileType) (errerror)

CopyFromOneLake uses the COPY INTO command to load data from OneLake

func (*MsFabricConn)GenerateDDL ¶added inv1.4.24

func (conn *MsFabricConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (string,error)

func (*MsFabricConn)Init ¶added inv1.4.24

func (conn *MsFabricConn) Init()error

Init initiates the object

typeMsSQLServerConn ¶

type MsSQLServerConn struct {BaseConnURLstring// contains filtered or unexported fields}

MsSQLServerConn is a Microsoft SQL Server connection

func (*MsSQLServerConn)BcpExport ¶

func (conn *MsSQLServerConn) BcpExport() (errerror)

BcpExport exports data to datastream

func (*MsSQLServerConn)BcpImportFile ¶

func (conn *MsSQLServerConn) BcpImportFile(tableFName, filePathstring) (countuint64, errerror)

BcpImportFile Import using bcp toolhttps://docs.microsoft.com/en-us/sql/tools/bcp-utility?view=sql-server-ver15bcp dbo.test1 in '/tmp/LargeDataset.csv' -S tcp:sqlserver.host,51433 -d master -U sa -P 'password' -c -t ',' -b 5000Limitation: if comma or delimite is in field, it will error.need to use delimiter not in field, or do some other transformation

func (*MsSQLServerConn)BcpImportFileParrallel ¶

func (conn *MsSQLServerConn) BcpImportFileParrallel(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BcpImportFileParrallel uses goroutine to import partitioned files

func (*MsSQLServerConn)BulkImportFlow ¶

func (conn *MsSQLServerConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow bulk import flow

func (*MsSQLServerConn)BulkImportStream ¶

func (conn *MsSQLServerConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream bulk import stream

func (*MsSQLServerConn)CastColumnForSelect ¶added inv1.3.5

func (conn *MsSQLServerConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*MsSQLServerConn)Close ¶added inv1.4.25

func (conn *MsSQLServerConn) Close()error

Close closes the SQL Server connection and cleans up Cloud SQL resources if applicable

func (*MsSQLServerConn)ConnString ¶added inv1.2.3

func (conn *MsSQLServerConn) ConnString()string

func (*MsSQLServerConn)Connect ¶added inv1.2.3

func (conn *MsSQLServerConn) Connect(timeOut ...int) (errerror)

func (*MsSQLServerConn)CopyFromAzure ¶

func (conn *MsSQLServerConn) CopyFromAzure(tableFName, azPathstring) (countuint64, errerror)

CopyFromAzure uses the COPY INTO Table command from Azurehttps://docs.microsoft.com/en-us/sql/t-sql/statements/copy-into-transact-sql?view=azure-sqldw-latest

func (*MsSQLServerConn)CopyViaAzure ¶

func (conn *MsSQLServerConn) CopyViaAzure(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

CopyViaAzure uses the Azure DWH COPY INTO Table command

func (*MsSQLServerConn)FedAuth ¶added inv1.4.24

func (conn *MsSQLServerConn) FedAuth()string

func (*MsSQLServerConn)GenerateDDL ¶added inv1.2.15

func (conn *MsSQLServerConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (string,error)

func (*MsSQLServerConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *MsSQLServerConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*MsSQLServerConn)GetTableColumns ¶added inv1.3.5

func (conn *MsSQLServerConn) GetTableColumns(table *Table, fields ...string) (columnsiop.Columns, errerror)

func (*MsSQLServerConn)GetURL ¶

func (conn *MsSQLServerConn) GetURL(newURL ...string)string

GetURL returns the processed URL

func (*MsSQLServerConn)Init ¶

func (conn *MsSQLServerConn) Init()error

Init initiates the object

func (*MsSQLServerConn)SubmitTemplate ¶added inv1.3.5

func (conn *MsSQLServerConn) SubmitTemplate(levelstring, templateMap map[string]string, namestring, values map[string]any) (dataiop.Dataset, errerror)

typeMySQLConn ¶

type MySQLConn struct {BaseConnURLstring// contains filtered or unexported fields}

MySQLConn is a MySQL or MariaDB connection

func (*MySQLConn)BulkExportStream ¶

func (conn *MySQLConn) BulkExportStream(tableTable) (ds *iop.Datastream, errerror)

BulkExportStream bulk Export

func (*MySQLConn)BulkImportStream ¶

func (conn *MySQLConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream bulk import stream

func (*MySQLConn)Close ¶added inv1.4.25

func (conn *MySQLConn) Close()error

Close closes the MySQL connection and cleans up Cloud SQL resources if applicable

func (*MySQLConn)Connect ¶added inv1.3.4

func (conn *MySQLConn) Connect(timeOut ...int) (errerror)

func (*MySQLConn)GenerateDDL ¶added inv1.2.15

func (conn *MySQLConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (string,error)

func (*MySQLConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *MySQLConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

UPSERThttps://vladmihalcea.com/how-do-upsert-and-merge-work-in-oracle-sql-server-postgresql-and-mysql/GenerateMergeSQL generates the upsert SQL

func (*MySQLConn)GetURL ¶

func (conn *MySQLConn) GetURL(newURL ...string)string

GetURL returns the processed URL

func (*MySQLConn)Init ¶

func (conn *MySQLConn) Init()error

Init initiates the object

func (*MySQLConn)LoadDataInFile ¶

func (conn *MySQLConn) LoadDataInFile(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

LoadDataInFile Bulk Import

func (*MySQLConn)LoadDataOutFile ¶

func (conn *MySQLConn) LoadDataOutFile(ctx *g.Context, sqlstring) (stdOutReaderio.Reader, errerror)

LoadDataOutFile Bulk ExportPossible error: ERROR 1227 (42000) at line 1: Access denied; you need (at least one of) the FILE privilege(s) for this operationFile privilege needs to be granted to useralso the --secure-file-priv option needs to be set properly for it to work.https://stackoverflow.com/questions/9819271/why-is-mysql-innodb-insert-so-slow to improve innodb insert speed

typeOperation ¶added inv1.4.24

type Operationstring

const (OperationMergeOperation = "merge"OperationDropTableOperation = "drop_table"OperationCreateIndexOperation = "create_index"OperationGenerateDataOperation = "generate_data")

typeOracleConn ¶

type OracleConn struct {BaseConnURLstring// contains filtered or unexported fields}

OracleConn is a Postgres connection

func (*OracleConn)BulkImportStream ¶

func (conn *OracleConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream bulk import stream

func (*OracleConn)CastColumnForSelect ¶added inv1.3.5

func (conn *OracleConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*OracleConn)ConnString ¶added inv1.1.14

func (conn *OracleConn) ConnString()string

func (*OracleConn)Connect ¶added inv1.1.14

func (conn *OracleConn) Connect(timeOut ...int) (errerror)

func (*OracleConn)ExecMultiContext ¶

func (conn *OracleConn) ExecMultiContext(ctxcontext.Context, qs ...string) (resultsql.Result, errerror)

ExecMultiContext runs multiple sql queries with context, returns `error`

func (*OracleConn)GenerateDDL ¶added inv1.2.15

func (conn *OracleConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (string,error)

func (*OracleConn)GenerateInsertStatement ¶

func (conn *OracleConn) GenerateInsertStatement(tableNamestring, colsiop.Columns, numRowsint)string

GenerateInsertStatement returns the proper INSERT statement

func (*OracleConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *OracleConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*OracleConn)GetTableColumns ¶added inv1.1.8

func (conn *OracleConn) GetTableColumns(table *Table, fields ...string) (columnsiop.Columns, errerror)

func (*OracleConn)Init ¶

func (conn *OracleConn) Init()error

Init initiates the object

func (*OracleConn)SQLLoad ¶

func (conn *OracleConn) SQLLoad(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

SQLLoad uses sqlldr to Bulk Importcat test1.csv | sqlldr system/oracle@oracle.host:1521/xe control=sqlldr.ctl log=/dev/stdout bad=/dev/stderrcannot import when newline in value. Need to scan for new lines.

func (*OracleConn)SubmitTemplate ¶added inv1.2.4

func (conn *OracleConn) SubmitTemplate(levelstring, templateMap map[string]string, namestring, values map[string]interface{}) (dataiop.Dataset, errerror)

func (*OracleConn)Version ¶added inv1.1.14

func (conn *OracleConn) Version()int

typePool ¶

type Pool struct {Dbs     map[string]*sqlx.DBDuckDbs map[string]*DuckDbConnMuxsync.Mutex}

Pool is a pool of connections

typePostgresConn ¶

type PostgresConn struct {BaseConnURLstring// contains filtered or unexported fields}

PostgresConn is a Postgres connection

func (*PostgresConn)BulkExportStream ¶

func (conn *PostgresConn) BulkExportStream(tableTable) (ds *iop.Datastream, errerror)

BulkExportStream uses the bulk dumping (COPY)

func (*PostgresConn)BulkImportStream ¶

func (conn *PostgresConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream inserts a stream into a table

func (*PostgresConn)CastColumnForSelect ¶

func (conn *PostgresConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*PostgresConn)Close ¶added inv1.4.25

func (conn *PostgresConn) Close()error

Close closes the database connection and cleans up Cloud SQL resources

func (*PostgresConn)Connect ¶added inv1.4.10

func (conn *PostgresConn) Connect(timeOut ...int)error

Connect connects to the database

func (*PostgresConn)CopyToStdout ¶

func (conn *PostgresConn) CopyToStdout(ctx *g.Context, sqlstring) (stdOutReaderio.Reader, errerror)

CopyToStdout Copy TO STDOUT

func (*PostgresConn)GenerateDDL ¶added inv1.1.4

func (conn *PostgresConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (ddlstring, errerror)

GenerateDDL generates a DDL based on a dataset

func (*PostgresConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *PostgresConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*PostgresConn)Init ¶

func (conn *PostgresConn) Init()error

Init initiates the object

typePrometheusConn ¶added inv1.2.2

type PrometheusConn struct {BaseConnURLstringClientv1.API}

PrometheusConn is a Prometheus connection

func (*PrometheusConn)BulkExportFlow ¶added inv1.2.2

func (conn *PrometheusConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

func (*PrometheusConn)Close ¶added inv1.2.2

func (conn *PrometheusConn) Close()error

func (*PrometheusConn)Connect ¶added inv1.2.2

func (conn *PrometheusConn) Connect(timeOut ...int)error

Connect connects to the database

func (*PrometheusConn)ExecContext ¶added inv1.2.2

func (conn *PrometheusConn) ExecContext(ctxcontext.Context, sqlstring, args ...interface{}) (resultsql.Result, errerror)

func (*PrometheusConn)GetSQLColumns ¶added inv1.2.2

func (conn *PrometheusConn) GetSQLColumns(tableTable) (columnsiop.Columns, errerror)

func (*PrometheusConn)GetSchemas ¶added inv1.2.2

func (conn *PrometheusConn) GetSchemas() (dataiop.Dataset, errerror)

GetSchemas returns schemas

func (*PrometheusConn)GetSchemata ¶added inv1.2.2

func (conn *PrometheusConn) GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (Schemata,error)

GetSchemata obtain full schemata info for a schema and/or table in current database

func (*PrometheusConn)GetTableColumns ¶added inv1.2.2

func (conn *PrometheusConn) GetTableColumns(table *Table, fields ...string) (columnsiop.Columns, errerror)

NewTransaction creates a new transaction

func (*PrometheusConn)GetTables ¶added inv1.2.2

func (conn *PrometheusConn) GetTables(schemastring) (dataiop.Dataset, errerror)

GetSchemas returns schemas

func (*PrometheusConn)Init ¶added inv1.2.2

func (conn *PrometheusConn) Init()error

Init initiates the object

func (*PrometheusConn)NewTransaction ¶added inv1.2.2

func (conn *PrometheusConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

func (*PrometheusConn)StreamRowsChunked ¶added inv1.4.11

func (conn *PrometheusConn) StreamRowsChunked(queryContext *g.Context, querystring, opts map[string]interface{}) (ds *iop.Datastream, errerror)

StreamRowsChunked implements chunked streaming for Prometheus to avoid memory issues

func (*PrometheusConn)StreamRowsContext ¶added inv1.2.2

func (conn *PrometheusConn) StreamRowsContext(ctxcontext.Context, querystring, Opts ...map[string]interface{}) (ds *iop.Datastream, errerror)

typeProtonConn ¶added inv1.2.7

type ProtonConn struct {BaseConnURLstring}

ProtonConn is a Proton connection

func (*ProtonConn)BulkImportStream ¶added inv1.2.7

func (conn *ProtonConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream inserts a stream into a table

func (*ProtonConn)ConnString ¶added inv1.2.7

func (conn *ProtonConn) ConnString()string

func (*ProtonConn)Connect ¶added inv1.2.7

func (conn *ProtonConn) Connect(timeOut ...int) (errerror)

func (*ProtonConn)ExecContext ¶added inv1.2.21

func (conn *ProtonConn) ExecContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)

ExecContext runs a sql query with context, returns `error`

func (*ProtonConn)GenerateDDL ¶added inv1.2.7

func (conn *ProtonConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (sqlstring, errerror)

GenerateDDL generates a DDL based on a dataset

func (*ProtonConn)GenerateInsertStatement ¶added inv1.2.7

func (conn *ProtonConn) GenerateInsertStatement(tableNamestring, colsiop.Columns, numRowsint)string

GenerateInsertStatement returns the proper INSERT statement

func (*ProtonConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *ProtonConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*ProtonConn)GetCount ¶added inv1.2.14

func (conn *ProtonConn) GetCount(tableFNamestring) (int64,error)

GetCount returns count of records

func (*ProtonConn)GetNativeType ¶added inv1.2.21

func (conn *ProtonConn) GetNativeType(coliop.Column) (nativeTypestring, errerror)

func (*ProtonConn)Init ¶added inv1.2.7

func (conn *ProtonConn) Init()error

Init initiates the object

func (*ProtonConn)NewTransaction ¶added inv1.2.7

func (conn *ProtonConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (Transaction,error)

NewTransaction creates a new transaction

typeRedshiftConn ¶

type RedshiftConn struct {BaseConnURLstring}

RedshiftConn is a Redshift connection

func (*RedshiftConn)BulkExportFlow ¶

func (conn *RedshiftConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

BulkExportFlow reads in bulk

func (*RedshiftConn)BulkExportStream ¶

func (conn *RedshiftConn) BulkExportStream(tableTable) (ds *iop.Datastream, errerror)

BulkExportStream reads in bulk

func (*RedshiftConn)BulkImportFlow ¶

func (conn *RedshiftConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow inserts a flow of streams into a table.For redshift we need to create CSVs in S3 and then use the COPY command.

func (*RedshiftConn)BulkImportStream ¶

func (conn *RedshiftConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream inserts a stream into a table.For redshift we need to create CSVs in S3 and then use the COPY command.

func (*RedshiftConn)CastColumnForSelect ¶added inv1.2.10

func (conn *RedshiftConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*RedshiftConn)ConnString ¶

func (conn *RedshiftConn) ConnString()string

func (*RedshiftConn)CopyFromS3 ¶

func (conn *RedshiftConn) CopyFromS3(tableFName, s3Pathstring, columnsiop.Columns) (countuint64, errerror)

CopyFromS3 uses the COPY INTO Table command from AWS S3

func (*RedshiftConn)GenerateDDL ¶added inv1.1.4

func (conn *RedshiftConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (sqlstring, errerror)

GenerateDDL generates a DDL based on a dataset

func (*RedshiftConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *RedshiftConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*RedshiftConn)Init ¶

func (conn *RedshiftConn) Init()error

Init initiates the object

func (*RedshiftConn)OptimizeTable ¶added inv1.1.9

func (conn *RedshiftConn) OptimizeTable(table *Table, newColumnsiop.Columns, isTemp ...bool) (okbool, errerror)

func (*RedshiftConn)Unload ¶

func (conn *RedshiftConn) Unload(ctx *g.Context, fileFormatdbio.FileType, tables ...Table) (s3Pathstring, errerror)

Unload unloads a query to S3

func (*RedshiftConn)WarnStlLoadErrors ¶added inv1.1.9

func (conn *RedshiftConn) WarnStlLoadErrors(errerror)

typeRelation ¶

type Relationstring

typeResult ¶

type Result struct {// contains filtered or unexported fields}

func (Result)LastInsertId ¶

func (rResult) LastInsertId() (int64,error)

func (Result)RowsAffected ¶

func (rResult) RowsAffected() (int64,error)

typeSQLiteConn ¶

type SQLiteConn struct {BaseConnURLstring}

SQLiteConn is a SQLite connection

func (*SQLiteConn)BulkImportStream ¶

func (conn *SQLiteConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream inserts a stream into a table

func (*SQLiteConn)GenerateDDL ¶added inv1.2.15

func (conn *SQLiteConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (string,error)

func (*SQLiteConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *SQLiteConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*SQLiteConn)GetSchemata ¶

func (conn *SQLiteConn) GetSchemata(levelSchemataLevel, schemaNamestring, tableNames ...string) (Schemata,error)

GetSchemata obtain full schemata info for a schema and/or table in current database

func (*SQLiteConn)GetURL ¶

func (conn *SQLiteConn) GetURL(newURL ...string)string

GetURL returns the processed URL

func (*SQLiteConn)Init ¶

func (conn *SQLiteConn) Init()error

Init initiates the object

typeSchema ¶

type Schema struct {Namestring           `json:"name"`Databasestring           `json:"database"`Tables   map[string]Table `json:"tables"`}

Schema represents a schemata schema

func (*Schema)Columns ¶

func (schema *Schema) Columns() map[string]iop.Column

func (*Schema)ToData ¶

func (schema *Schema) ToData() (dataiop.Dataset)

ToData converts schema objects to tabular format

typeSchemata ¶

type Schemata struct {Databases map[string]Database `json:"databases"`// contains filtered or unexported fields}

Schemata contains the full schema for a connection

funcGetSchemataAll ¶

func GetSchemataAll(connConnection) (schemataSchemata, errerror)

GetSchemataAll obtains the schemata for all databases detected

funcGetTablesSchemata ¶

func GetTablesSchemata(connConnection, tableNames ...string) (schemataSchemata, errerror)

GetTablesSchemata obtains the schemata for specified tables

func (*Schemata)Columns ¶

func (s *Schemata) Columns(filters ...string) map[string]iop.Column

func (*Schemata)Database ¶

func (s *Schemata) Database()Database

Database returns the first encountered database

func (*Schemata)Filtered ¶added inv1.1.15

func (s *Schemata) Filtered(columnLevelbool, filters ...string) (nsSchemata)

func (*Schemata)LoadTablesJSON ¶

func (s *Schemata) LoadTablesJSON(payloadstring)error

LoadTablesJSON loads from a json string

func (*Schemata)Tables ¶

func (s *Schemata) Tables(filters ...string) map[string]Table

typeSchemataLevel ¶added inv1.2.19

type SchemataLevelstring

const (SchemataLevelSchemaSchemataLevel = "schema"SchemataLevelTableSchemataLevel = "table"SchemataLevelColumnSchemataLevel = "column")

typeSelectOptions ¶added inv1.3.5

type SelectOptions struct {Fields []stringOffsetintLimit  *intWherestring}

typeSnowflakeConn ¶

type SnowflakeConn struct {BaseConnURLstringWarehousestringCopyMethodstringMuxsync.Mutex}

SnowflakeConn is a Snowflake connection

func (*SnowflakeConn)BulkExportFlow ¶

func (conn *SnowflakeConn) BulkExportFlow(tableTable) (df *iop.Dataflow, errerror)

BulkExportFlow reads in bulk

func (*SnowflakeConn)BulkImportFlow ¶

func (conn *SnowflakeConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow bulk import flow

func (*SnowflakeConn)BulkImportStream ¶

func (conn *SnowflakeConn) BulkImportStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

BulkImportStream bulk import stream

func (*SnowflakeConn)CastColumnForSelect ¶added inv1.1.15

func (conn *SnowflakeConn) CastColumnForSelect(srcColiop.Column, tgtColiop.Column) (selectStrstring)

CastColumnForSelect casts to the correct target column type

func (*SnowflakeConn)ConnString ¶

func (conn *SnowflakeConn) ConnString()string

func (*SnowflakeConn)Connect ¶

func (conn *SnowflakeConn) Connect(timeOut ...int)error

Connect connects to the database

func (*SnowflakeConn)CopyFromAzure ¶

func (conn *SnowflakeConn) CopyFromAzure(tableFName, azPathstring) (errerror)

CopyFromAzure uses the Snowflake COPY INTO Table command from Azurehttps://docs.snowflake.com/en/sql-reference/sql/copy-into-table.html

func (*SnowflakeConn)CopyFromS3 ¶

func (conn *SnowflakeConn) CopyFromS3(tableFName, s3Pathstring) (errerror)

CopyFromS3 uses the Snowflake COPY INTO Table command from AWS S3https://docs.snowflake.com/en/sql-reference/sql/copy-into-table.html

func (*SnowflakeConn)CopyToAzure ¶

func (conn *SnowflakeConn) CopyToAzure(tables ...Table) (azPathstring, errerror)

CopyToAzure exports a query to an Azure location

func (*SnowflakeConn)CopyToS3 ¶

func (conn *SnowflakeConn) CopyToS3(tables ...Table) (s3Pathstring, errerror)

CopyToS3 exports a query to an S3 location

func (*SnowflakeConn)CopyViaAWS ¶

func (conn *SnowflakeConn) CopyViaAWS(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

CopyViaAWS uses the Snowflake COPY INTO Table command from AWS S3https://docs.snowflake.com/en/sql-reference/sql/copy-into-table.html

func (*SnowflakeConn)CopyViaAzure ¶

func (conn *SnowflakeConn) CopyViaAzure(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

CopyViaAzure uses the Snowflake COPY INTO Table command from Azurehttps://docs.snowflake.com/en/sql-reference/sql/copy-into-table.html

func (*SnowflakeConn)CopyViaStage ¶

func (conn *SnowflakeConn) CopyViaStage(tableTable, df *iop.Dataflow) (countuint64, errerror)

CopyViaStage uses the Snowflake COPY INTO Table commandhttps://docs.snowflake.com/en/sql-reference/sql/copy-into-table.html

func (*SnowflakeConn)GenerateDDL ¶added inv1.1.4

func (conn *SnowflakeConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (sqlstring, errerror)

GenerateDDL generates a DDL based on a dataset

func (*SnowflakeConn)GenerateInsertStatement ¶added inv1.2.19

func (conn *SnowflakeConn) GenerateInsertStatement(tableNamestring, colsiop.Columns, numRowsint)string

func (*SnowflakeConn)GenerateMergeSQL ¶added inv1.4.20

func (conn *SnowflakeConn) GenerateMergeSQL(srcTablestring, tgtTablestring, pkFields []string) (sqlstring, errerror)

GenerateMergeSQL generates the upsert SQL

func (*SnowflakeConn)GetColumnsFull ¶

func (conn *SnowflakeConn) GetColumnsFull(tableFNamestring) (dataiop.Dataset, errerror)

func (*SnowflakeConn)GetDatabases ¶

func (conn *SnowflakeConn) GetDatabases() (dataiop.Dataset, errerror)

GetDatabases returns the list of databases

func (*SnowflakeConn)GetSchemas ¶

func (conn *SnowflakeConn) GetSchemas() (dataiop.Dataset, errerror)

GetSchemas returns schemas

func (*SnowflakeConn)GetTables ¶

func (conn *SnowflakeConn) GetTables(schemastring) (dataiop.Dataset, errerror)

GetTables returns tables

func (*SnowflakeConn)GetTablesAndViews ¶added inv1.2.19

func (conn *SnowflakeConn) GetTablesAndViews(schemastring) (iop.Dataset,error)

GetTablesAndViews returns tables/views for given schema

func (*SnowflakeConn)GetViews ¶

func (conn *SnowflakeConn) GetViews(schemastring) (dataiop.Dataset, errerror)

GetTables returns tables

func (*SnowflakeConn)Init ¶

func (conn *SnowflakeConn) Init()error

Init initiates the object

func (*SnowflakeConn)StageGET ¶added inv1.2.14

func (conn *SnowflakeConn) StageGET(internalStagePath, folderPathstring) (filePaths []string, errerror)

StageGET Copies from a staging location to a local file or folder

func (*SnowflakeConn)StagePUT ¶added inv1.2.14

func (conn *SnowflakeConn) StagePUT(fileURIstring, internalStagePathstring) (errerror)

StagePUT Copies a local file or folder into a staging location

func (*SnowflakeConn)UnloadViaStage ¶

func (conn *SnowflakeConn) UnloadViaStage(formatdbio.FileType, tables ...Table) (filePathstring, unloadedint64, errerror)

typeSqlServerLogger ¶added inv1.3.6

type SqlServerLogger struct{}

func (*SqlServerLogger)Printf ¶added inv1.3.6

func (l *SqlServerLogger) Printf(formatstring, v ...any)

func (*SqlServerLogger)Println ¶added inv1.3.6

func (l *SqlServerLogger) Println(v ...any)

typeStarRocksConn ¶

type StarRocksConn struct {BaseConnURLstring// contains filtered or unexported fields}

StarRocksConn is a StarRocks connection

func (*StarRocksConn)AddMissingColumns ¶added inv1.1.8

func (conn *StarRocksConn) AddMissingColumns(tableTable, newColsiop.Columns) (okbool, errerror)

func (*StarRocksConn)BulkImportFlow ¶

func (conn *StarRocksConn) BulkImportFlow(tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

BulkImportFlow inserts a flow of streams into a table.

func (*StarRocksConn)Connect ¶added inv1.4.16

func (conn *StarRocksConn) Connect(timeOut ...int) (errerror)

func (*StarRocksConn)ExecContext ¶added inv1.2.19

func (conn *StarRocksConn) ExecContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)

ExecContext runs a sql query with context, returns `error`

func (*StarRocksConn)GenerateDDL ¶

func (conn *StarRocksConn) GenerateDDL(tableTable, dataiop.Dataset, temporarybool) (string,error)

GenerateDDL generates a DDL based on a dataset

func (*StarRocksConn)GetDatabases ¶added inv1.1.9

func (conn *StarRocksConn) GetDatabases() (dataiop.Dataset, errerror)

GetDatabases returns the list of databases

func (*StarRocksConn)GetURL ¶

func (conn *StarRocksConn) GetURL(newURL ...string)string

GetURL returns the processed URL

func (*StarRocksConn)Init ¶

func (conn *StarRocksConn) Init()error

Init initiates the object

func (*StarRocksConn)InsertBatchStream ¶

func (conn *StarRocksConn) InsertBatchStream(tableFNamestring, ds *iop.Datastream) (countuint64, errerror)

InsertBatchStream inserts a stream into a table in batch

func (*StarRocksConn)NewTransaction ¶added inv1.4.25

func (conn *StarRocksConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

func (*StarRocksConn)OptimizeTable ¶added inv1.1.8

func (conn *StarRocksConn) OptimizeTable(table *Table, newColumnsiop.Columns, isTemp ...bool) (okbool, errerror)

func (*StarRocksConn)StreamLoad ¶

func (conn *StarRocksConn) StreamLoad(feURL, tableFNamestring, df *iop.Dataflow) (countuint64, errerror)

StreamLoad bulk loadshttps://docs.starrocks.io/docs/loading/StreamLoad/https://docs.starrocks.io/docs/sql-reference/sql-statements/data-manipulation/STREAM_LOAD/

func (*StarRocksConn)WaitAlterTable ¶added inv1.1.8

func (conn *StarRocksConn) WaitAlterTable(tableTable) (errerror)

typeStatFieldSQL ¶

type StatFieldSQL struct {NamestringTemplateSQLstring}

typeTable ¶

type Table struct {Namestring      `json:"name"`Schemastring      `json:"schema"`Databasestring      `json:"database,omitempty"`IsViewbool        `json:"is_view,omitempty"`// whether is a viewSQLstring      `json:"sql,omitempty"`DDLstring      `json:"ddl,omitempty"`Dialectdbio.Type   `json:"dialect,omitempty"`Columnsiop.Columns `json:"columns,omitempty"`KeysTableKeys   `json:"keys,omitempty"`Rawstring `json:"raw"`// contains filtered or unexported fields}

Table represents a schemata table

funcParseTableName ¶

func ParseTableName(textstring, dialectdbio.Type) (tableTable, errerror)

func (*Table)AddPrimaryKeyToDDL ¶added inv1.2.15

func (t *Table) AddPrimaryKeyToDDL(ddlstring, columnsiop.Columns) (string,error)

AddPrimaryKeyToDDL adds a primary key to the table

func (*Table)Clone ¶added inv1.2.14

func (t *Table) Clone()Table

func (*Table)ColumnsMap ¶

func (t *Table) ColumnsMap() map[string]iop.Column

func (*Table)DatabaseQ ¶

func (t *Table) DatabaseQ()string

func (*Table)FDQN ¶

func (t *Table) FDQN()string

func (*Table)FullName ¶

func (t *Table) FullName()string

func (*Table)Indexes ¶added inv1.2.15

func (t *Table) Indexes(columnsiop.Columns) (indexes []TableIndex)

func (*Table)IsProcedural ¶added inv1.3.5

func (t *Table) IsProcedural()bool

func (*Table)IsQuery ¶

func (t *Table) IsQuery()bool

func (*Table)MarshalJSON ¶added inv1.2.19

func (t *Table) MarshalJSON() ([]byte,error)

func (*Table)NameQ ¶

func (t *Table) NameQ()string

func (*Table)SchemaQ ¶

func (t *Table) SchemaQ()string

func (*Table)Select ¶

func (t *Table) Select(Opts ...SelectOptions) (sqlstring)

func (*Table)SetKeys ¶added inv1.1.8

func (t *Table) SetKeys(sourcePKCols []string, updateColstring, tableKeysTableKeys)error

typeTableIndex ¶added inv1.2.15

type TableIndex struct {NamestringColumnsiop.ColumnsUniqueboolTable   *Table}

func (*TableIndex)CreateDDL ¶added inv1.2.15

func (ti *TableIndex) CreateDDL()string

func (*TableIndex)DropDDL ¶added inv1.2.15

func (ti *TableIndex) DropDDL()string

typeTableKeys ¶added inv1.1.4

type TableKeys map[iop.KeyType][]string

typeTransaction ¶

type Transaction interface {Connection()ConnectionContext() *g.ContextCommit() (errerror)Rollback() (errerror)Prepare(querystring) (stmt *sql.Stmt, errerror)QueryContext(ctxcontext.Context, qstring, args ...interface{}) (result *sqlx.Rows, errerror)ExecContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)ExecMultiContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)}

typeTrinoConn ¶added inv1.1.14

type TrinoConn struct {BaseConnURLstring}

TrinoConn is a Trino connection

func (*TrinoConn)ConnString ¶added inv1.1.14

func (conn *TrinoConn) ConnString()string

func (*TrinoConn)ExecContext ¶added inv1.1.14

func (conn *TrinoConn) ExecContext(ctxcontext.Context, qstring, args ...interface{}) (resultsql.Result, errerror)

ExecContext runs a sql query with context, returns `error`

func (*TrinoConn)Init ¶added inv1.1.14

func (conn *TrinoConn) Init()error

Init initiates the object

func (*TrinoConn)NewTransaction ¶added inv1.1.14

func (conn *TrinoConn) NewTransaction(ctxcontext.Context, options ...*sql.TxOptions) (txTransaction, errerror)

NewTransaction creates a new transaction

typeUser ¶

type User struct {ModelDbXNamestringAgeint}

funcNewUser ¶

func NewUser() *User

typeWhereClause ¶

type WhereClause []interface{}

WhereClause is the where clause

func (WhereClause)Args ¶

func (wcWhereClause) Args() []interface{}

Args returns the where clause arguments

func (WhereClause)Clause ¶

func (wcWhereClause) Clause()string

Clause returns the string where clause

Source Files¶

View all Source files

?	: This menu
/	: Search site
f orF	: Jump to
y orY	: Canonical URL

Movatterモバイル変換

Details

Repository

Links

Documentation¶

Index¶

Constants¶

Variables¶

Functions¶

funcChangeColumnTypeViaAdd¶

funcCommonColumns¶

funcCopyFromAzure¶

funcCopyFromS3¶

funcEnsureBinSQLite¶

funcGenerateAlterDDL¶

funcGetArrowDBCDriverType¶added inv1.5.2

funcGetOptimizeTableStatements¶added inv1.1.8

funcGetQualifierQuote¶

funcInsertBatchStream¶

funcInsertStream¶

funcMakeDuckDbSecretProps¶added inv1.4.11

funcMerge¶added inv1.4.20

funcNativeTypeToGeneral¶

funcPK¶

funcParseColumnName¶

funcParseSQLMultiStatements¶

funcPrintSessionID¶added inv1.4.24

funcQueryOperation¶added inv1.4.24

funcSQLColumns¶

funcSplitTableFullName¶

funcTestPermissions¶

funcTrimSQLComments¶added inv1.4.7

funcTruncateTable¶added inv1.4.14

funcUID¶

Types¶

typeArrowDBConn¶added inv1.5.2

func (*ArrowDBConn)BulkExportFlow¶added inv1.5.2

func (*ArrowDBConn)BulkExportStream¶added inv1.5.2

func (*ArrowDBConn)BulkImportFlow¶added inv1.5.2

func (*ArrowDBConn)BulkImportStream¶added inv1.5.2

func (*ArrowDBConn)Close¶added inv1.5.2

func (*ArrowDBConn)Connect¶added inv1.5.2

func (*ArrowDBConn)ExecContext¶added inv1.5.2

func (*ArrowDBConn)GetNativeType¶added inv1.5.2

func (*ArrowDBConn)GetSQLColumns¶added inv1.5.2

func (*ArrowDBConn)GetTemplateValue¶added inv1.5.2

func (*ArrowDBConn)Init¶added inv1.5.2

func (*ArrowDBConn)LoadTemplates¶added inv1.5.2

func (*ArrowDBConn)Quote¶added inv1.5.2

func (*ArrowDBConn)StreamRowsContext¶added inv1.5.2

func (*ArrowDBConn)Template¶added inv1.5.2

func (*ArrowDBConn)Unquote¶added inv1.5.2

typeAthenaConn¶added inv1.4.7

func (*AthenaConn)BulkExportFlow¶added inv1.4.7

func (*AthenaConn)BulkExportStream¶added inv1.4.7

func (*AthenaConn)BulkImportFlow¶added inv1.4.7

func (*AthenaConn)BulkImportStream¶added inv1.4.7

func (*AthenaConn)CastColumnForSelect¶added inv1.4.7

func (*AthenaConn)Close¶added inv1.4.7

func (*AthenaConn)Connect¶added inv1.4.7

func (*AthenaConn)ExecContext¶added inv1.4.7

func (*AthenaConn)GenerateDDL¶added inv1.4.7

func (*AthenaConn)GenerateMergeSQL¶added inv1.4.20

func (*AthenaConn)GetDatabases¶added inv1.4.7

func (*AthenaConn)GetSchemata¶added inv1.4.7

func (*AthenaConn)Init¶added inv1.4.7

func (*AthenaConn)InsertBatchStream¶added inv1.4.7

func (*AthenaConn)InsertStream¶added inv1.4.7

func (*AthenaConn)LoadFromS3¶added inv1.4.10

func (*AthenaConn)NewTransaction¶added inv1.4.7

func (*AthenaConn)StagingBucket¶added inv1.4.10

func (*AthenaConn)StreamRowsContext¶added inv1.4.7

func (*AthenaConn)Unload¶added inv1.4.7

typeAzureTableConn¶added inv1.4.16

func (*AzureTableConn)BulkExportFlow¶added inv1.4.16

func (*AzureTableConn)BulkImportFlow¶added inv1.4.16

func (*AzureTableConn)Close¶added inv1.4.16

func (*AzureTableConn)Connect¶added inv1.4.16

func (*AzureTableConn)ExecContext¶added inv1.4.16

func (*AzureTableConn)GetCount¶added inv1.4.16

Documentation ¶

funcChangeColumnTypeViaAdd ¶

funcCommonColumns ¶

funcCopyFromAzure ¶

funcCopyFromS3 ¶

funcEnsureBinSQLite ¶

funcGenerateAlterDDL ¶

funcGetArrowDBCDriverType ¶added inv1.5.2

funcGetOptimizeTableStatements ¶added inv1.1.8

funcGetQualifierQuote ¶

funcInsertBatchStream ¶

funcInsertStream ¶

funcMakeDuckDbSecretProps ¶added inv1.4.11

funcMerge ¶added inv1.4.20

funcNativeTypeToGeneral ¶

funcPK ¶

funcParseColumnName ¶

funcParseSQLMultiStatements ¶

funcPrintSessionID ¶added inv1.4.24

funcQueryOperation ¶added inv1.4.24

funcSQLColumns ¶

funcSplitTableFullName ¶

funcTestPermissions ¶

funcTrimSQLComments ¶added inv1.4.7

funcTruncateTable ¶added inv1.4.14

funcUID ¶

typeArrowDBConn ¶added inv1.5.2

func (*ArrowDBConn)BulkExportFlow ¶added inv1.5.2

func (*ArrowDBConn)BulkExportStream ¶added inv1.5.2

func (*ArrowDBConn)BulkImportFlow ¶added inv1.5.2

func (*ArrowDBConn)BulkImportStream ¶added inv1.5.2

func (*ArrowDBConn)Close ¶added inv1.5.2

func (*ArrowDBConn)Connect ¶added inv1.5.2

func (*ArrowDBConn)ExecContext ¶added inv1.5.2

func (*ArrowDBConn)GetNativeType ¶added inv1.5.2

func (*ArrowDBConn)GetSQLColumns ¶added inv1.5.2

func (*ArrowDBConn)GetTemplateValue ¶added inv1.5.2

func (*ArrowDBConn)Init ¶added inv1.5.2

func (*ArrowDBConn)LoadTemplates ¶added inv1.5.2

func (*ArrowDBConn)Quote ¶added inv1.5.2

func (*ArrowDBConn)StreamRowsContext ¶added inv1.5.2

func (*ArrowDBConn)Template ¶added inv1.5.2

func (*ArrowDBConn)Unquote ¶added inv1.5.2

typeAthenaConn ¶added inv1.4.7

func (*AthenaConn)BulkExportFlow ¶added inv1.4.7

func (*AthenaConn)BulkExportStream ¶added inv1.4.7

func (*AthenaConn)BulkImportFlow ¶added inv1.4.7

func (*AthenaConn)BulkImportStream ¶added inv1.4.7

func (*AthenaConn)CastColumnForSelect ¶added inv1.4.7

func (*AthenaConn)Close ¶added inv1.4.7

func (*AthenaConn)Connect ¶added inv1.4.7

func (*AthenaConn)ExecContext ¶added inv1.4.7

func (*AthenaConn)GenerateDDL ¶added inv1.4.7

func (*AthenaConn)GenerateMergeSQL ¶added inv1.4.20

func (*AthenaConn)GetDatabases ¶added inv1.4.7

func (*AthenaConn)GetSchemata ¶added inv1.4.7

func (*AthenaConn)Init ¶added inv1.4.7

func (*AthenaConn)InsertBatchStream ¶added inv1.4.7

func (*AthenaConn)InsertStream ¶added inv1.4.7

func (*AthenaConn)LoadFromS3 ¶added inv1.4.10

func (*AthenaConn)NewTransaction ¶added inv1.4.7

func (*AthenaConn)StagingBucket ¶added inv1.4.10

func (*AthenaConn)StreamRowsContext ¶added inv1.4.7

func (*AthenaConn)Unload ¶added inv1.4.7

typeAzureTableConn ¶added inv1.4.16

func (*AzureTableConn)BulkExportFlow ¶added inv1.4.16

func (*AzureTableConn)BulkImportFlow ¶added inv1.4.16

func (*AzureTableConn)Close ¶added inv1.4.16

func (*AzureTableConn)Connect ¶added inv1.4.16

func (*AzureTableConn)ExecContext ¶added inv1.4.16

func (*AzureTableConn)GetCount ¶added inv1.4.16

func (*AzureTableConn)GetSchemas ¶added inv1.4.16

func (*AzureTableConn)GetSchemata ¶added inv1.4.16

func (*AzureTableConn)GetTableColumns ¶added inv1.4.16

func (*AzureTableConn)GetTables ¶added inv1.4.16

func (*AzureTableConn)Init ¶added inv1.4.16

func (*AzureTableConn)NewTransaction ¶added inv1.4.16

func (*AzureTableConn)StreamRowsContext ¶added inv1.4.16

func (*AzureTableConn)TableExists ¶added inv1.4.16

typeAzureTableResult ¶added inv1.4.16