Movatterモバイル変換

[0]ホーム
URL:

filesys

package
v1.5.1Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest Published: Dec 7, 2025 License:GPL-3.0Imports:58Imported by:2

Details

Repository

github.com/slingdata-io/sling-cli

Links

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

funcCopyFromLocalRecursiveadded inv1.2.25

func CopyFromLocalRecursive(fsFileSysClient, localPathstring, remotePathstring) (totalBytesint64, errerror)

CopyFromLocalRecursive copies a local file or directory recursively to a remote filesystem

funcCopyFromRemoteRecursiveadded inv1.3.5

func CopyFromRemoteRecursive(fsFileSysClient, remotePathstring, localPathstring) (totalBytesint64, errerror)

CopyFromRemoteRecursive copies a remote file or directory recursively to a local filesystem

funcCopyRecursiveadded inv1.3.6

func CopyRecursive(fromFs, toFsFileSysClient, fromPath, toPathstring) (totalBytesint64, errerror)

CopyRecursive copies from/to remote systems by streaming. The folderstructure should be maintained. The read/write operations should be done concurrently.

funcDelete

func Delete(fsFileSysClient, uristring) (errerror)

Delete deletes the provided path before writingwith some safeguards so to not accidentally delete some root path

funcGetDataflow

func GetDataflow(fsFileSysClient, nodesFileNodes, cfgiop.FileStreamConfig) (df *iop.Dataflow, errerror)

GetDataflow returns a dataflow from specified paths in specified FileSysClient

funcGetDataflowViaDuckDBadded inv1.3.5

func GetDataflowViaDuckDB(fsFileSysClient, uristring, nodesFileNodes, cfgiop.FileStreamConfig) (df *iop.Dataflow, errerror)

GetDataflowViaDuckDB returns a dataflow from specified paths in specified FileSysClient

funcGetDeepestParentadded inv1.1.15

func GetDeepestParent(pathstring)string

funcGetDeepestPartitionParentadded inv1.3.6

func GetDeepestPartitionParent(pathstring)string

funcGetFirstDatePartURIadded inv1.3.6

func GetFirstDatePartURI(fsFileSysClient, maskstring) (uristring, errerror)

GetFirstDatePartURI determines the first part for the URI mask provided

funcInferFileFormat

func InferFileFormat(pathstring, defaults ...dbio.FileType)dbio.FileType

funcMakeDatastream

func MakeDatastream(readerio.Reader, cfg map[string]string) (ds *iop.Datastream, errerror)

MakeDatastream create a datastream from a reader

funcMergeReaders

func MergeReaders(fsFileSysClient, fileTypedbio.FileType, nodesFileNodes, cfgiop.FileStreamConfig) (ds *iop.Datastream, errerror)

funcNormalizeURIadded inv1.1.15

func NormalizeURI(fsFileSysClient, uristring)string

funcParseURL

func ParseURL(uristring) (host, pathstring, errerror)

ParseURL parses a URL

funcParseURLTypeadded inv1.2.16

func ParseURLType(uristring) (uTypedbio.Type, hoststring, pathstring, errerror)

ParseURL parses a URL

funcPeekFileType

func PeekFileType(readerio.Reader) (ftdbio.FileType, reader2io.Reader, errerror)

PeekFileType peeks into the file to try determine the file typeCSV is the default

funcTestFsPermissions

func TestFsPermissions(fsFileSysClient, pathURLstring) (errerror)

TestFsPermissions tests read/write permisions

funcWrite

func Write(readerio.Reader, writerio.Writer) (bwint64, errerror)

Write writer to a writer from a reader

funcWriteDataflowadded inv1.2.10

func WriteDataflow(fsFileSysClient, df *iop.Dataflow, urlstring) (bwint64, errerror)

WriteDataflow writes a dataflow to a file sys.

funcWriteDataflowReadyViaDuckDBadded inv1.4.24

func WriteDataflowReadyViaDuckDB(fsFileSysClient, df *iop.Dataflow, uristring, fileReadyChn chanFileReady, sciop.StreamConfig) (bwint64, errerror)

WriteDataflowViaDuckDB writes to parquet / csv via duckdb

funcWriteDataflowViaDuckDBadded inv1.3.6

func WriteDataflowViaDuckDB(fsFileSysClient, df *iop.Dataflow, uristring) (bwint64, errerror)

Types

typeABFSFileSysClientadded inv1.4.24

type ABFSFileSysClient struct {BaseFileSysClient// contains filtered or unexported fields}

ABFSFileSysClient is a file system client to write file to Azure Data Lake Storage Gen2 (ABFS)

func (*ABFSFileSysClient)Bucketsadded inv1.4.24

func (fs *ABFSFileSysClient) Buckets() (paths []string, errerror)

Buckets returns the filesystems found in the account

func (*ABFSFileSysClient)Connectadded inv1.4.24

func (fs *ABFSFileSysClient) Connect() (errerror)

Connect initiates the fs client connection

func (*ABFSFileSysClient)GetPathadded inv1.4.24

func (fs *ABFSFileSysClient) GetPath(uristring) (pathstring, errerror)

GetPath returns the path of url

func (*ABFSFileSysClient)GetReaderadded inv1.4.24

func (fs *ABFSFileSysClient) GetReader(uristring) (readerio.Reader, errerror)

GetReader returns an ABFS reader

func (*ABFSFileSysClient)Initadded inv1.4.24

func (fs *ABFSFileSysClient) Init(ctxcontext.Context) (errerror)

Init initializes the fs client

func (*ABFSFileSysClient)Listadded inv1.4.24

func (fs *ABFSFileSysClient) List(uristring) (nodesFileNodes, errerror)

List list objects in path (non-recursive)

func (*ABFSFileSysClient)ListRecursiveadded inv1.4.24

func (fs *ABFSFileSysClient) ListRecursive(uristring) (nodesFileNodes, errerror)

ListRecursive list objects in path recursively

func (*ABFSFileSysClient)Prefixadded inv1.4.24

func (fs *ABFSFileSysClient) Prefix(suffix ...string)string

Prefix returns the url prefix

func (*ABFSFileSysClient)Writeadded inv1.4.24

func (fs *ABFSFileSysClient) Write(uristring, readerio.Reader) (bwint64, errerror)

Write writes a stream to a file

typeAzureFileSysClient

type AzureFileSysClient struct {BaseFileSysClient// contains filtered or unexported fields}

AzureFileSysClient is a file system client to write file to Microsoft's Azure file sys.

func (*AzureFileSysClient)Buckets

func (fs *AzureFileSysClient) Buckets() (paths []string, errerror)

Buckets returns the containers found in the project

func (*AzureFileSysClient)Connect

func (fs *AzureFileSysClient) Connect() (errerror)

Connect initiates the fs client connection

func (*AzureFileSysClient)GetPathadded inv1.1.15

func (fs *AzureFileSysClient) GetPath(uristring) (pathstring, errerror)

GetPath returns the path of url

func (*AzureFileSysClient)GetReader

func (fs *AzureFileSysClient) GetReader(uristring) (readerio.Reader, errerror)

GetReader returns an Azure FS reader

func (*AzureFileSysClient)Init

func (fs *AzureFileSysClient) Init(ctxcontext.Context) (errerror)

Init initializes the fs client

func (*AzureFileSysClient)List

func (fs *AzureFileSysClient) List(uristring) (nodesFileNodes, errerror)

List list objects in path

func (*AzureFileSysClient)ListRecursive

func (fs *AzureFileSysClient) ListRecursive(uristring) (nodesFileNodes, errerror)

ListRecursive list objects in path

func (*AzureFileSysClient)Prefixadded inv1.1.15

func (fs *AzureFileSysClient) Prefix(suffix ...string)string

Prefix returns the url prefix

func (*AzureFileSysClient)Write

func (fs *AzureFileSysClient) Write(uristring, readerio.Reader) (bwint64, errerror)

typeBaseFileSysClient

type BaseFileSysClient struct {FileSysClient// contains filtered or unexported fields}

BaseFileSysClient is the base file system type.

func (*BaseFileSysClient)Buckets

func (fs *BaseFileSysClient) Buckets() (paths []string, errerror)

Buckets returns the buckets found in the account

func (*BaseFileSysClient)Client

Client provides a pointer to itself

func (*BaseFileSysClient)Closeadded inv1.2.12

func (fs *BaseFileSysClient) Close()error

Close closes the client

func (*BaseFileSysClient)Context

func (fs *BaseFileSysClient) Context() (context *g.Context)

Context provides a pointer to context

func (*BaseFileSysClient)FsType

func (fs *BaseFileSysClient) FsType()dbio.Type

FsType return the type of the client

func (*BaseFileSysClient)GetDatastream

func (fs *BaseFileSysClient) GetDatastream(uristring, cfg ...iop.FileStreamConfig) (ds *iop.Datastream, errerror)

GetDatastream return a datastream for the given path

func (*BaseFileSysClient)GetProp

func (fs *BaseFileSysClient) GetProp(keystring, keys ...string)string

GetProp returns the value of a property

func (*BaseFileSysClient)GetReaders

func (fs *BaseFileSysClient) GetReaders(paths ...string) (readers []io.Reader, errerror)

GetReaders returns one or more readers from specified paths in specified FileSysClient

func (*BaseFileSysClient)GetRefTs

func (fs *BaseFileSysClient) GetRefTs()time.Time

func (*BaseFileSysClient)Prefixadded inv1.1.15

func (fs *BaseFileSysClient) Prefix(suffix ...string)string

Prefix returns the url prefix

func (*BaseFileSysClient)Props

func (fs *BaseFileSysClient) Props() map[string]string

Props returns a copy of the properties map

func (*BaseFileSysClient)Queryadded inv1.2.16

func (fs *BaseFileSysClient) Query(uri, sqlstring) (dataiop.Dataset, errerror)

Query queries the file system via duckdb

func (*BaseFileSysClient)ReadDataflow

func (fs *BaseFileSysClient) ReadDataflow(urlstring, cfg ...iop.FileStreamConfig) (df *iop.Dataflow, errerror)

ReadDataflow read

func (*BaseFileSysClient)Self

Instance returns the respective connection InstanceThis is useful to refer back to a subclass methodfrom the superclass level. (Aka overloading)

func (*BaseFileSysClient)SetProp

func (fs *BaseFileSysClient) SetProp(keystring, valstring)

SetProp sets the value of a property

func (*BaseFileSysClient)WriteDataflowReady

func (fs *BaseFileSysClient) WriteDataflowReady(df *iop.Dataflow, urlstring, fileReadyChn chanFileReady, sciop.StreamConfig) (bwint64, errerror)

WriteDataflowReady writes to a file sys and notifies the fileReady chan.

typeFileNodeadded inv1.2.16

type FileNode struct {URIstring      `json:"uri"`IsDirbool        `json:"is_dir"`Sizeuint64      `json:"size,omitempty"`Createdint64       `json:"created,omitempty"`Updatedint64       `json:"updated,omitempty"`Ownerstring      `json:"owner,omitempty"`Columnsiop.Columns `json:"columns,omitempty"`ChildrenFileNodes   `json:"children,omitempty"`// contains filtered or unexported fields}

FileNode represents a file node

func (*FileNode)MarshalJSONadded inv1.2.19

func (fn *FileNode) MarshalJSON() ([]byte,error)

func (*FileNode)Nameadded inv1.2.16

func (fn *FileNode) Name()string

func (*FileNode)Pathadded inv1.2.16

func (fn *FileNode) Path()string

Path returns the path without the prefix

func (*FileNode)Typeadded inv1.2.16

func (fn *FileNode) Type()dbio.Type

Type return the file type

typeFileNodesadded inv1.2.16

type FileNodes []FileNode

FileNodes represent file nodes

funcCopyFromRemoteNodesadded inv1.3.5

func CopyFromRemoteNodes(fsFileSysClient, urlstring, nodesFileNodes, localRootstring) (totalBytesint64, localNodesFileNodes, errerror)

CopyFromRemoteNodes copies the nodes to local path maintaining the same folder structure

funcNewFileNodesadded inv1.2.16

func NewFileNodes(nodes []map[string]any) (fnsFileNodes)

NewFileNodes creates fileNodes from slice of maps

func (*FileNodes)Addadded inv1.2.16

func (fns *FileNodes) Add(ns ...FileNode)

Add adds a new node to list

func (*FileNodes)AddWhereadded inv1.2.16

func (fns *FileNodes) AddWhere(pattern *glob.Glob, afterint64, ns ...FileNode)

AddWhere adds a new node to list if pattern matches name and after timestamp

func (FileNodes)Afteradded inv1.2.16

func (fnsFileNodes) After(tstime.Time) (nodesFileNodes)

After returns the nodes modified after provided time

func (FileNodes)AvgSizeadded inv1.2.16

func (fnsFileNodes) AvgSize()uint64

func (FileNodes)Columnsadded inv1.2.16

func (fnsFileNodes) Columns() map[string]iop.Column

Columns returns a column map

func (FileNodes)Filesadded inv1.3.4

func (fnsFileNodes) Files() (nodesFileNodes)

Files returns only files (no folders)

func (FileNodes)Foldersadded inv1.3.4

func (fnsFileNodes) Folders() (nodesFileNodes)

Folders returns only folders (no files)

func (FileNodes)InferFormatadded inv1.3.5

func (fnsFileNodes) InferFormat() (formatdbio.FileType)

InferFormat returns the most common file format

func (FileNodes)Pathsadded inv1.2.16

func (fnsFileNodes) Paths() (paths []string)

Paths give a list of recursive paths

func (FileNodes)SelectWithPrefixadded inv1.3.6

func (fnsFileNodes) SelectWithPrefix(prefixes ...string) (nodesFileNodes)

SelectWithPrefix returns only nodes with one of the prefixes

func (FileNodes)Sortadded inv1.2.16

func (fnsFileNodes) Sort()

Sort sorts the nodes

func (FileNodes)TotalSizeadded inv1.2.16

func (fnsFileNodes) TotalSize()uint64

func (FileNodes)URIsadded inv1.2.16

func (fnsFileNodes) URIs() (uris []string)

URIs give a list of recursive uris

typeFileReady

type FileReady struct {Columnsiop.ColumnsNodeFileNodeBytesWint64BatchIDstring}

typeFileSysClient

type FileSysClient interface {Self()FileSysClientInit(ctxcontext.Context) (errerror)Close() (errerror)Client() *BaseFileSysClientContext() (context *g.Context)FsType()dbio.TypeGetReader(pathstring) (readerio.Reader, errerror)GetReaders(paths ...string) (readers []io.Reader, errerror)GetDatastream(pathstring, cfg ...iop.FileStreamConfig) (ds *iop.Datastream, errerror)GetWriter(pathstring) (writerio.Writer, errerror)Buckets() (paths []string, errerror)List(pathstring) (pathsFileNodes, errerror)ListRecursive(pathstring) (pathsFileNodes, errerror)Write(pathstring, readerio.Reader) (bwint64, errerror)Prefix(suffix ...string)stringReadDataflow(urlstring, cfg ...iop.FileStreamConfig) (df *iop.Dataflow, errerror)WriteDataflowReady(df *iop.Dataflow, urlstring, fileReadyChn chanFileReady, sciop.StreamConfig) (bwint64, errerror)GetProp(keystring, keys ...string) (valstring)SetProp(keystring, valstring)Props() map[string]stringMkdirAll(pathstring) (errerror)GetPath(uristring) (pathstring, errerror)Query(uri, sqlstring) (dataiop.Dataset, errerror)// contains filtered or unexported methods}

FileSysClient is a client to a file systemssuch as local, s3, hdfs, azure storage, google cloud storage

funcNewFileSysClient

func NewFileSysClient(fstdbio.Type, props ...string) (fsClientFileSysClient, errerror)

NewFileSysClient create a file system clientsuch as local, s3, azure storage, google cloud storageprops are provided as `"Prop1=Value1", "Prop2=Value2", ...`

funcNewFileSysClientContext

func NewFileSysClientContext(ctxcontext.Context, fstdbio.Type, props ...string) (fsClientFileSysClient, errerror)

NewFileSysClientContext create a file system client with contextsuch as local, s3, azure storage, google cloud storageprops are provided as `"Prop1=Value1", "Prop2=Value2", ...`

funcNewFileSysClientFromURL

func NewFileSysClientFromURL(urlstring, props ...string) (fsClientFileSysClient, errerror)

NewFileSysClientFromURL returns the proper fs client for the given pathprops are provided as `"Prop1=Value1", "Prop2=Value2", ...`

funcNewFileSysClientFromURLContext

func NewFileSysClientFromURLContext(ctxcontext.Context, urlstring, props ...string) (fsClientFileSysClient, errerror)

NewFileSysClientFromURLContext returns the proper fs client for the given path with contextprops are provided as `"Prop1=Value1", "Prop2=Value2", ...`

typeFtpFileSysClientadded inv1.1.12

type FtpFileSysClient struct {BaseFileSysClient// contains filtered or unexported fields}

FtpFileSysClient is for FTP file systems

func (*FtpFileSysClient)Closeadded inv1.1.12

func (fs *FtpFileSysClient) Close()error

func (*FtpFileSysClient)Connectadded inv1.1.12

func (fs *FtpFileSysClient) Connect() (errerror)

Connect initiates the Google Cloud Storage client

func (*FtpFileSysClient)GetPathadded inv1.1.15

func (fs *FtpFileSysClient) GetPath(uristring) (pathstring, errerror)

GetPath returns the path of url

func (*FtpFileSysClient)GetReaderadded inv1.1.12

func (fs *FtpFileSysClient) GetReader(urlStrstring) (readerio.Reader, errerror)

GetReader return a reader for the given path

func (*FtpFileSysClient)Initadded inv1.1.12

func (fs *FtpFileSysClient) Init(ctxcontext.Context) (errerror)

Init initializes the fs client

func (*FtpFileSysClient)Listadded inv1.1.12

func (fs *FtpFileSysClient) List(urlstring) (nodesFileNodes, errerror)

List list objects in path

func (*FtpFileSysClient)ListRecursiveadded inv1.1.12

func (fs *FtpFileSysClient) ListRecursive(urlstring) (nodesFileNodes, errerror)

ListRecursive list objects in path recursively

func (*FtpFileSysClient)MkdirAlladded inv1.1.12

func (fs *FtpFileSysClient) MkdirAll(pathstring) (errerror)

MkdirAll creates child directories

func (*FtpFileSysClient)Prefixadded inv1.1.15

func (fs *FtpFileSysClient) Prefix(suffix ...string)string

Prefix returns the url prefix

func (*FtpFileSysClient)Reconnectadded inv1.2.12

func (fs *FtpFileSysClient) Reconnect(reasonstring)

func (*FtpFileSysClient)Writeadded inv1.1.12

func (fs *FtpFileSysClient) Write(urlStrstring, readerio.Reader) (bwint64, errerror)

typeGoogleDriveFileSysClientadded inv1.4.10

type GoogleDriveFileSysClient struct {BaseFileSysClient// contains filtered or unexported fields}

GoogleDriveFileSysClient is a file system client for Google Drive

func (*GoogleDriveFileSysClient)Bucketsadded inv1.4.10

func (fs *GoogleDriveFileSysClient) Buckets() (paths []string, errerror)

Buckets returns the root folder (Google Drive doesn't have buckets)

func (*GoogleDriveFileSysClient)Connectadded inv1.4.10

func (fs *GoogleDriveFileSysClient) Connect() (errerror)

Connect initiates the Google Drive client

func (*GoogleDriveFileSysClient)GetPathadded inv1.4.10

func (fs *GoogleDriveFileSysClient) GetPath(uristring) (pathstring, errerror)

GetPath returns the path of url

func (*GoogleDriveFileSysClient)GetReaderadded inv1.4.10

func (fs *GoogleDriveFileSysClient) GetReader(uristring) (readerio.Reader, errerror)

GetReader returns a reader for the given path

func (*GoogleDriveFileSysClient)Initadded inv1.4.10

Init initializes the fs client

func (*GoogleDriveFileSysClient)Listadded inv1.4.10

func (fs *GoogleDriveFileSysClient) List(uristring) (nodesFileNodes, errerror)

List returns the list of objects

func (*GoogleDriveFileSysClient)ListRecursiveadded inv1.4.10

func (fs *GoogleDriveFileSysClient) ListRecursive(uristring) (nodesFileNodes, errerror)

ListRecursive returns the list of objects recursively

func (*GoogleDriveFileSysClient)MkdirAlladded inv1.4.10

func (fs *GoogleDriveFileSysClient) MkdirAll(pathstring) (errerror)

MkdirAll creates a directory path and all parents that don't exist

func (*GoogleDriveFileSysClient)Prefixadded inv1.4.10

func (fs *GoogleDriveFileSysClient) Prefix(suffix ...string)string

Prefix returns the url prefix

func (*GoogleDriveFileSysClient)Writeadded inv1.4.10

func (fs *GoogleDriveFileSysClient) Write(uristring, readerio.Reader) (bwint64, errerror)

Write writes data to Google Drive

typeGoogleFileSysClient

type GoogleFileSysClient struct {BaseFileSysClient// contains filtered or unexported fields}

GoogleFileSysClient is a file system client to write file to Amazon's S3 file sys.

func (*GoogleFileSysClient)Buckets

func (fs *GoogleFileSysClient) Buckets() (paths []string, errerror)

Buckets returns the buckets found in the project

func (*GoogleFileSysClient)Connect

func (fs *GoogleFileSysClient) Connect() (errerror)

Connect initiates the Google Cloud Storage client

func (*GoogleFileSysClient)GetPathadded inv1.1.15

func (fs *GoogleFileSysClient) GetPath(uristring) (pathstring, errerror)

GetPath returns the path of url

func (*GoogleFileSysClient)GetReader

func (fs *GoogleFileSysClient) GetReader(pathstring) (readerio.Reader, errerror)

GetReader returns the reader for the given path

func (*GoogleFileSysClient)Init

func (fs *GoogleFileSysClient) Init(ctxcontext.Context) (errerror)

Init initializes the fs client

func (*GoogleFileSysClient)List

func (fs *GoogleFileSysClient) List(uristring) (nodesFileNodes, errerror)

List returns the list of objects

func (*GoogleFileSysClient)ListRecursive

func (fs *GoogleFileSysClient) ListRecursive(uristring) (nodesFileNodes, errerror)

ListRecursive returns the list of objects recursively

func (*GoogleFileSysClient)Prefixadded inv1.1.15

func (fs *GoogleFileSysClient) Prefix(suffix ...string)string

Prefix returns the url prefix

func (*GoogleFileSysClient)Write

func (fs *GoogleFileSysClient) Write(pathstring, readerio.Reader) (bwint64, errerror)

typeHTTPFileSysClient

type HTTPFileSysClient struct {BaseFileSysClient// contains filtered or unexported fields}

HTTPFileSysClient is for HTTP files

func (*HTTPFileSysClient)Connect

func (fs *HTTPFileSysClient) Connect() (errerror)

Connect initiates the http client

func (*HTTPFileSysClient)GetPathadded inv1.1.15

func (fs *HTTPFileSysClient) GetPath(uristring) (pathstring, errerror)

GetPath returns the path of url

func (*HTTPFileSysClient)GetReader

func (fs *HTTPFileSysClient) GetReader(urlstring) (readerio.Reader, errerror)

GetReader gets a reader for an HTTP resource (download)

func (*HTTPFileSysClient)Init

func (fs *HTTPFileSysClient) Init(ctxcontext.Context) (errerror)

Init initializes the fs client

func (*HTTPFileSysClient)List

func (fs *HTTPFileSysClient) List(urlstring) (nodesFileNodes, errerror)

List lists all urls on the page

func (*HTTPFileSysClient)ListRecursive

func (fs *HTTPFileSysClient) ListRecursive(urlstring) (nodesFileNodes, errerror)

ListRecursive lists all urls on the page

func (*HTTPFileSysClient)Prefixadded inv1.1.15

func (fs *HTTPFileSysClient) Prefix(suffix ...string)string

Prefix returns the url prefix

func (*HTTPFileSysClient)Write

func (fs *HTTPFileSysClient) Write(urlStrstring, readerio.Reader) (bwint64, errerror)

Write uploads an HTTP file

typeLocalFileSysClient

type LocalFileSysClient struct {BaseFileSysClient// contains filtered or unexported fields}

LocalFileSysClient is a file system client to write file to local file sys.

func (*LocalFileSysClient)GetDatastream

func (fs *LocalFileSysClient) GetDatastream(uristring, cfg ...iop.FileStreamConfig) (ds *iop.Datastream, errerror)

GetDatastream return a datastream for the given path

func (*LocalFileSysClient)GetPathadded inv1.1.15

func (fs *LocalFileSysClient) GetPath(uristring) (pathstring, errerror)

GetPath returns the path of url

func (*LocalFileSysClient)GetReader

func (fs *LocalFileSysClient) GetReader(uristring) (readerio.Reader, errerror)

GetReader return a reader for the given path

func (*LocalFileSysClient)GetWriter

func (fs *LocalFileSysClient) GetWriter(uristring) (writerio.Writer, errerror)

GetWriter creates the file if non-existent and return a writer

func (*LocalFileSysClient)Init

func (fs *LocalFileSysClient) Init(ctxcontext.Context) (errerror)

Init initializes the fs client

func (*LocalFileSysClient)List

func (fs *LocalFileSysClient) List(uristring) (nodesFileNodes, errerror)

List lists the file in given directory path

func (*LocalFileSysClient)ListRecursive

func (fs *LocalFileSysClient) ListRecursive(uristring) (nodesFileNodes, errerror)

ListRecursive lists the file in given directory path recursively

func (*LocalFileSysClient)MkdirAll

func (fs *LocalFileSysClient) MkdirAll(uristring) (errerror)

MkdirAll creates child directories

func (*LocalFileSysClient)Prefixadded inv1.1.15

func (fs *LocalFileSysClient) Prefix(suffix ...string)string

Prefix returns the url prefix

func (*LocalFileSysClient)Write

func (fs *LocalFileSysClient) Write(uristring, readerio.Reader) (bwint64, errerror)

Write creates the file if non-existent and writes from the reader

typeS3FileSysClient

type S3FileSysClient struct {BaseFileSysClientRegionMap map[string]string// contains filtered or unexported fields}

S3FileSysClient is a file system client to write file to Amazon's S3 file sys.

func (*S3FileSysClient)Buckets

func (fs *S3FileSysClient) Buckets() (paths []string, errerror)

Buckets returns the buckets found in the account

func (*S3FileSysClient)Connect

func (fs *S3FileSysClient) Connect() (errerror)

Connect initiates the S3 client

func (*S3FileSysClient)GenerateS3PreSignedURL

func (fs *S3FileSysClient) GenerateS3PreSignedURL(s3URLstring, durtime.Duration) (httpURLstring, errerror)

func (*S3FileSysClient)GetPathadded inv1.1.15

func (fs *S3FileSysClient) GetPath(uristring) (pathstring, errerror)

GetPath returns the path of url

func (*S3FileSysClient)GetReader

func (fs *S3FileSysClient) GetReader(uristring) (readerio.Reader, errerror)

GetReader return a reader for the given pathpath should specify the full path with scheme:`s3://my_bucket/key/to/file.txt` or `s3://my_bucket/key/to/directory`

func (*S3FileSysClient)GetWriter

func (fs *S3FileSysClient) GetWriter(uristring) (writerio.Writer, errerror)

GetWriter creates the file if non-existent and return a writerpath should specify the full path with scheme:`s3://my_bucket/key/to/file.txt`

func (*S3FileSysClient)Init

func (fs *S3FileSysClient) Init(ctxcontext.Context) (errerror)

Init initializes the fs client

func (*S3FileSysClient)List

func (fs *S3FileSysClient) List(uristring) (nodesFileNodes, errerror)

List lists the file in given directory path

func (*S3FileSysClient)ListRecursive

func (fs *S3FileSysClient) ListRecursive(uristring) (nodesFileNodes, errerror)

ListRecursive lists the file in given directory path recusivelypath should specify the full path with scheme:`s3://my_bucket/key/to/directory`

func (*S3FileSysClient)Prefixadded inv1.1.15

func (fs *S3FileSysClient) Prefix(suffix ...string)string

Prefix returns the url prefix

func (*S3FileSysClient)Write

func (fs *S3FileSysClient) Write(uristring, readerio.Reader) (bwint64, errerror)

typeS3ReaderWrapperadded inv1.4.13

type S3ReaderWrapper struct {io.ReadCloser// contains filtered or unexported fields}

S3ReaderWrapper wraps an io.ReadCloser and ensures proper cleanup

func (*S3ReaderWrapper)Closeadded inv1.4.13

func (r *S3ReaderWrapper) Close()error

Close ensures the underlying ReadCloser is closed only once

func (*S3ReaderWrapper)Readadded inv1.4.13

func (r *S3ReaderWrapper) Read(p []byte) (nint, errerror)

Read implements io.Reader and automatically closes on EOF

typeSftpFileSysClient

type SftpFileSysClient struct {BaseFileSysClient// contains filtered or unexported fields}

SftpFileSysClient is for SFTP / SSH file ops

func (*SftpFileSysClient)Closeadded inv1.2.12

func (fs *SftpFileSysClient) Close()error

Close closes the client

func (*SftpFileSysClient)Connect

func (fs *SftpFileSysClient) Connect() (errerror)

Connect initiates the Google Cloud Storage client

func (*SftpFileSysClient)GetPathadded inv1.1.15

func (fs *SftpFileSysClient) GetPath(uristring) (pathstring, errerror)

GetPath returns the path of url

func (*SftpFileSysClient)GetReader

func (fs *SftpFileSysClient) GetReader(urlStrstring) (readerio.Reader, errerror)

GetReader return a reader for the given path

func (*SftpFileSysClient)GetWriter

func (fs *SftpFileSysClient) GetWriter(urlStrstring) (writerio.Writer, errerror)

GetWriter creates the file if non-existent and return a writer

func (*SftpFileSysClient)Init

func (fs *SftpFileSysClient) Init(ctxcontext.Context) (errerror)

Init initializes the fs client

func (*SftpFileSysClient)List

func (fs *SftpFileSysClient) List(urlstring) (nodesFileNodes, errerror)

List list objects in path

func (*SftpFileSysClient)ListRecursive

func (fs *SftpFileSysClient) ListRecursive(uristring) (nodesFileNodes, errerror)

ListRecursive list objects in path recursively

func (*SftpFileSysClient)MkdirAll

func (fs *SftpFileSysClient) MkdirAll(pathstring) (errerror)

MkdirAll creates child directories

func (*SftpFileSysClient)Prefixadded inv1.1.15

func (fs *SftpFileSysClient) Prefix(suffix ...string)string

Prefix returns the url prefix

func (*SftpFileSysClient)Write

func (fs *SftpFileSysClient) Write(urlStrstring, readerio.Reader) (bwint64, errerror)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f orF : Jump to
y orY : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic.Learn more.
[8]ページ先頭
©2009-2025 Movatter.jp