23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99100101102103104105106107108109110111112113114115116

defconnect(uri:URI,*,api_key:Optional[str]=None,region:str="us-east-1",host_override:Optional[str]=None,read_consistency_interval:Optional[timedelta]=None,request_thread_pool:Optional[Union[int,ThreadPoolExecutor]]=None,client_config:Union[ClientConfig,Dict[str,Any],None]=None,storage_options:Optional[Dict[str,str]]=None,**kwargs:Any,)->DBConnection:"""Connect to a LanceDB database.    Parameters    ----------    uri: str or Path        The uri of the database.    api_key: str, optional        If presented, connect to LanceDB cloud.        Otherwise, connect to a database on file system or cloud storage.        Can be set via environment variable `LANCEDB_API_KEY`.    region: str, default "us-east-1"        The region to use for LanceDB Cloud.    host_override: str, optional        The override url for LanceDB Cloud.    read_consistency_interval: timedelta, default None        (For LanceDB OSS only)        The interval at which to check for updates to the table from other        processes. If None, then consistency is not checked. For performance        reasons, this is the default. For strong consistency, set this to        zero seconds. Then every read will check for updates from other        processes. As a compromise, you can set this to a non-zero timedelta        for eventual consistency. If more than that interval has passed since        the last check, then the table will be checked for updates. Note: this        consistency only applies to read operations. Write operations are        always consistent.    client_config: ClientConfig or dict, optional        Configuration options for the LanceDB Cloud HTTP client. If a dict, then        the keys are the attributes of the ClientConfig class. If None, then the        default configuration is used.    storage_options: dict, optional        Additional options for the storage backend. See available options at        <https://lancedb.github.io/lancedb/guides/storage/>    Examples    --------    For a local directory, provide a path for the database:    >>> import lancedb    >>> db = lancedb.connect("~/.lancedb")    For object storage, use a URI prefix:    >>> db = lancedb.connect("s3://my-bucket/lancedb",    ...                      storage_options={"aws_access_key_id": "***"})    Connect to LanceDB cloud:    >>> db = lancedb.connect("db://my_database", api_key="ldb_...",    ...                      client_config={"retry_config": {"retries": 5}})    Returns    -------    conn : DBConnection        A connection to a LanceDB database.    """ifisinstance(uri,str)anduri.startswith("db://"):ifapi_keyisNone:api_key=os.environ.get("LANCEDB_API_KEY")ifapi_keyisNone:raiseValueError(f"api_key is required to connected LanceDB cloud:{uri}")ifisinstance(request_thread_pool,int):request_thread_pool=ThreadPoolExecutor(request_thread_pool)returnRemoteDBConnection(uri,api_key,region,host_override,# TODO: remove this (deprecation warning downstream)request_thread_pool=request_thread_pool,client_config=client_config,storage_options=storage_options,**kwargs,)ifkwargs:raiseValueError(f"Unknown keyword arguments:{kwargs}")returnLanceDBConnection(uri,read_consistency_interval=read_consistency_interval,storage_options=storage_options,)

 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308

classDBConnection(EnforceOverrides):"""An active LanceDB connection interface."""@abstractmethoddeftable_names(self,page_token:Optional[str]=None,limit:int=10)->Iterable[str]:"""List all tables in this database, in sorted order        Parameters        ----------        page_token: str, optional            The token to use for pagination. If not present, start from the beginning.            Typically, this token is last table name from the previous page.            Only supported by LanceDb Cloud.        limit: int, default 10            The size of the page to return.            Only supported by LanceDb Cloud.        Returns        -------        Iterable of str        """pass@abstractmethoddefcreate_table(self,name:str,data:Optional[DATA]=None,schema:Optional[Union[pa.Schema,LanceModel]]=None,mode:str="create",exist_ok:bool=False,on_bad_vectors:str="error",fill_value:float=0.0,embedding_functions:Optional[List[EmbeddingFunctionConfig]]=None,*,storage_options:Optional[Dict[str,str]]=None,data_storage_version:Optional[str]=None,enable_v2_manifest_paths:Optional[bool]=None,)->Table:"""Create a [Table][lancedb.table.Table] in the database.        Parameters        ----------        name: str            The name of the table.        data: The data to initialize the table, *optional*            User must provide at least one of `data` or `schema`.            Acceptable types are:            - list-of-dict            - pandas.DataFrame            - pyarrow.Table or pyarrow.RecordBatch        schema: The schema of the table, *optional*            Acceptable types are:            - pyarrow.Schema            - [LanceModel][lancedb.pydantic.LanceModel]        mode: str; default "create"            The mode to use when creating the table.            Can be either "create" or "overwrite".            By default, if the table already exists, an exception is raised.            If you want to overwrite the table, use mode="overwrite".        exist_ok: bool, default False            If a table by the same name already exists, then raise an exception            if exist_ok=False. If exist_ok=True, then open the existing table;            it will not add the provided data but will validate against any            schema that's specified.        on_bad_vectors: str, default "error"            What to do if any of the vectors are not the same size or contains NaNs.            One of "error", "drop", "fill".        fill_value: float            The value to use when filling vectors. Only used if on_bad_vectors="fill".        storage_options: dict, optional            Additional options for the storage backend. Options already set on the            connection will be inherited by the table, but can be overridden here.            See available options at            <https://lancedb.github.io/lancedb/guides/storage/>        data_storage_version: optional, str, default "stable"            Deprecated.  Set `storage_options` when connecting to the database and set            `new_table_data_storage_version` in the options.        enable_v2_manifest_paths: optional, bool, default False            Deprecated.  Set `storage_options` when connecting to the database and set            `new_table_enable_v2_manifest_paths` in the options.        Returns        -------        LanceTable            A reference to the newly created table.        !!! note            The vector index won't be created by default.            To create the index, call the `create_index` method on the table.        Examples        --------        Can create with list of tuples or dictionaries:        >>> import lancedb        >>> db = lancedb.connect("./.lancedb")        >>> data = [{"vector": [1.1, 1.2], "lat": 45.5, "long": -122.7},        ...         {"vector": [0.2, 1.8], "lat": 40.1, "long":  -74.1}]        >>> db.create_table("my_table", data)        LanceTable(name='my_table', version=1, ...)        >>> db["my_table"].head()        pyarrow.Table        vector: fixed_size_list<item: float>[2]          child 0, item: float        lat: double        long: double        ----        vector: [[[1.1,1.2],[0.2,1.8]]]        lat: [[45.5,40.1]]        long: [[-122.7,-74.1]]        You can also pass a pandas DataFrame:        >>> import pandas as pd        >>> data = pd.DataFrame({        ...    "vector": [[1.1, 1.2], [0.2, 1.8]],        ...    "lat": [45.5, 40.1],        ...    "long": [-122.7, -74.1]        ... })        >>> db.create_table("table2", data)        LanceTable(name='table2', version=1, ...)        >>> db["table2"].head()        pyarrow.Table        vector: fixed_size_list<item: float>[2]          child 0, item: float        lat: double        long: double        ----        vector: [[[1.1,1.2],[0.2,1.8]]]        lat: [[45.5,40.1]]        long: [[-122.7,-74.1]]        Data is converted to Arrow before being written to disk. For maximum        control over how data is saved, either provide the PyArrow schema to        convert to or else provide a [PyArrow Table](pyarrow.Table) directly.        >>> import pyarrow as pa        >>> custom_schema = pa.schema([        ...   pa.field("vector", pa.list_(pa.float32(), 2)),        ...   pa.field("lat", pa.float32()),        ...   pa.field("long", pa.float32())        ... ])        >>> db.create_table("table3", data, schema = custom_schema)        LanceTable(name='table3', version=1, ...)        >>> db["table3"].head()        pyarrow.Table        vector: fixed_size_list<item: float>[2]          child 0, item: float        lat: float        long: float        ----        vector: [[[1.1,1.2],[0.2,1.8]]]        lat: [[45.5,40.1]]        long: [[-122.7,-74.1]]        It is also possible to create an table from `[Iterable[pa.RecordBatch]]`:        >>> import pyarrow as pa        >>> def make_batches():        ...     for i in range(5):        ...         yield pa.RecordBatch.from_arrays(        ...             [        ...                 pa.array([[3.1, 4.1], [5.9, 26.5]],        ...                     pa.list_(pa.float32(), 2)),        ...                 pa.array(["foo", "bar"]),        ...                 pa.array([10.0, 20.0]),        ...             ],        ...             ["vector", "item", "price"],        ...         )        >>> schema=pa.schema([        ...     pa.field("vector", pa.list_(pa.float32(), 2)),        ...     pa.field("item", pa.utf8()),        ...     pa.field("price", pa.float32()),        ... ])        >>> db.create_table("table4", make_batches(), schema=schema)        LanceTable(name='table4', version=1, ...)        """raiseNotImplementedErrordef__getitem__(self,name:str)->LanceTable:returnself.open_table(name)defopen_table(self,name:str,*,storage_options:Optional[Dict[str,str]]=None,index_cache_size:Optional[int]=None,)->Table:"""Open a Lance Table in the database.        Parameters        ----------        name: str            The name of the table.        index_cache_size: int, default 256            Set the size of the index cache, specified as a number of entries            The exact meaning of an "entry" will depend on the type of index:            * IVF - there is one entry for each IVF partition            * BTREE - there is one entry for the entire index            This cache applies to the entire opened table, across all indices.            Setting this value higher will increase performance on larger datasets            at the expense of more RAM        storage_options: dict, optional            Additional options for the storage backend. Options already set on the            connection will be inherited by the table, but can be overridden here.            See available options at            <https://lancedb.github.io/lancedb/guides/storage/>        Returns        -------        A LanceTable object representing the table.        """raiseNotImplementedErrordefdrop_table(self,name:str):"""Drop a table from the database.        Parameters        ----------        name: str            The name of the table.        """raiseNotImplementedErrordefrename_table(self,cur_name:str,new_name:str):"""Rename a table in the database.        Parameters        ----------        cur_name: str            The current name of the table.        new_name: str            The new name of the table.        """raiseNotImplementedErrordefdrop_database(self):"""        Drop database        This is the same thing as dropping all the tables        """raiseNotImplementedErrordefdrop_all_tables(self):"""        Drop all tables from the database        """raiseNotImplementedError@propertydefuri(self)->str:returnself._uri

 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 9991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549

classTable(ABC):"""    A Table is a collection of Records in a LanceDB Database.    Examples    --------    Create using [DBConnection.create_table][lancedb.DBConnection.create_table]    (more examples in that method's documentation).    >>> import lancedb    >>> db = lancedb.connect("./.lancedb")    >>> table = db.create_table("my_table", data=[{"vector": [1.1, 1.2], "b": 2}])    >>> table.head()    pyarrow.Table    vector: fixed_size_list<item: float>[2]      child 0, item: float    b: int64    ----    vector: [[[1.1,1.2]]]    b: [[2]]    Can append new data with [Table.add()][lancedb.table.Table.add].    >>> table.add([{"vector": [0.5, 1.3], "b": 4}])    AddResult(version=2)    Can query the table with [Table.search][lancedb.table.Table.search].    >>> table.search([0.4, 0.4]).select(["b", "vector"]).to_pandas()       b      vector  _distance    0  4  [0.5, 1.3]       0.82    1  2  [1.1, 1.2]       1.13    Search queries are much faster when an index is created. See    [Table.create_index][lancedb.table.Table.create_index].    """@property@abstractmethoddefname(self)->str:"""The name of this Table"""raiseNotImplementedError@property@abstractmethoddefversion(self)->int:"""The version of this Table"""raiseNotImplementedError@property@abstractmethoddefschema(self)->pa.Schema:"""The [Arrow Schema](https://arrow.apache.org/docs/python/api/datatypes.html#)        of this Table        """raiseNotImplementedError@property@abstractmethoddeftags(self)->Tags:"""Tag management for the table.        Similar to Git, tags are a way to add metadata to a specific version of the        table.        .. warning::            Tagged versions are exempted from the :py:meth:`cleanup_old_versions()`            process.            To remove a version that has been tagged, you must first            :py:meth:`~Tags.delete` the associated tag.        Examples        --------        .. code-block:: python            table = db.open_table("my_table")            table.tags.create("v2-prod-20250203", 10)            tags = table.tags.list()        """raiseNotImplementedErrordef__len__(self)->int:"""The number of rows in this Table"""returnself.count_rows(None)@property@abstractmethoddefembedding_functions(self)->Dict[str,EmbeddingFunctionConfig]:"""        Get a mapping from vector column name to it's configured embedding function.        """@abstractmethoddefcount_rows(self,filter:Optional[str]=None)->int:"""        Count the number of rows in the table.        Parameters        ----------        filter: str, optional            A SQL where clause to filter the rows to count.        """raiseNotImplementedErrordefto_pandas(self)->"pandas.DataFrame":"""Return the table as a pandas DataFrame.        Returns        -------        pd.DataFrame        """returnself.to_arrow().to_pandas()@abstractmethoddefto_arrow(self)->pa.Table:"""Return the table as a pyarrow Table.        Returns        -------        pa.Table        """raiseNotImplementedErrordefcreate_index(self,metric="l2",num_partitions=256,num_sub_vectors=96,vector_column_name:str=VECTOR_COLUMN_NAME,replace:bool=True,accelerator:Optional[str]=None,index_cache_size:Optional[int]=None,*,index_type:VectorIndexType="IVF_PQ",wait_timeout:Optional[timedelta]=None,num_bits:int=8,max_iterations:int=50,sample_rate:int=256,m:int=20,ef_construction:int=300,):"""Create an index on the table.        Parameters        ----------        metric: str, default "l2"            The distance metric to use when creating the index.            Valid values are "l2", "cosine", "dot", or "hamming".            l2 is euclidean distance.            Hamming is available only for binary vectors.        num_partitions: int, default 256            The number of IVF partitions to use when creating the index.            Default is 256.        num_sub_vectors: int, default 96            The number of PQ sub-vectors to use when creating the index.            Default is 96.        vector_column_name: str, default "vector"            The vector column name to create the index.        replace: bool, default True            - If True, replace the existing index if it exists.            - If False, raise an error if duplicate index exists.        accelerator: str, default None            If set, use the given accelerator to create the index.            Only support "cuda" for now.        index_cache_size : int, optional            The size of the index cache in number of entries. Default value is 256.        num_bits: int            The number of bits to encode sub-vectors. Only used with the IVF_PQ index.            Only 4 and 8 are supported.        wait_timeout: timedelta, optional            The timeout to wait if indexing is asynchronous.        """raiseNotImplementedErrordefdrop_index(self,name:str)->None:"""        Drop an index from the table.        Parameters        ----------        name: str            The name of the index to drop.        Notes        -----        This does not delete the index from disk, it just removes it from the table.        To delete the index, run [optimize][lancedb.table.Table.optimize]        after dropping the index.        Use [list_indices][lancedb.table.Table.list_indices] to find the names of        the indices.        """raiseNotImplementedErrordefwait_for_index(self,index_names:Iterable[str],timeout:timedelta=timedelta(seconds=300))->None:"""        Wait for indexing to complete for the given index names.        This will poll the table until all the indices are fully indexed,        or raise a timeout exception if the timeout is reached.        Parameters        ----------        index_names: str            The name of the indices to poll        timeout: timedelta            Timeout to wait for asynchronous indexing. The default is 5 minutes.        """raiseNotImplementedError@abstractmethoddefstats(self)->TableStatistics:"""        Retrieve table and fragment statistics.        """raiseNotImplementedError@abstractmethoddefcreate_scalar_index(self,column:str,*,replace:bool=True,index_type:ScalarIndexType="BTREE",wait_timeout:Optional[timedelta]=None,):"""Create a scalar index on a column.        Parameters        ----------        column : str            The column to be indexed.  Must be a boolean, integer, float,            or string column.        replace : bool, default True            Replace the existing index if it exists.        index_type: Literal["BTREE", "BITMAP", "LABEL_LIST"], default "BTREE"            The type of index to create.        wait_timeout: timedelta, optional            The timeout to wait if indexing is asynchronous.        Examples        --------        Scalar indices, like vector indices, can be used to speed up scans.  A scalar        index can speed up scans that contain filter expressions on the indexed column.        For example, the following scan will be faster if the column ``my_col`` has        a scalar index:        >>> import lancedb # doctest: +SKIP        >>> db = lancedb.connect("/data/lance") # doctest: +SKIP        >>> img_table = db.open_table("images") # doctest: +SKIP        >>> my_df = img_table.search().where("my_col = 7", # doctest: +SKIP        ...                                  prefilter=True).to_pandas()        Scalar indices can also speed up scans containing a vector search and a        prefilter:        >>> import lancedb # doctest: +SKIP        >>> db = lancedb.connect("/data/lance") # doctest: +SKIP        >>> img_table = db.open_table("images") # doctest: +SKIP        >>> img_table.search([1, 2, 3, 4], vector_column_name="vector") # doctest: +SKIP        ...     .where("my_col != 7", prefilter=True)        ...     .to_pandas()        Scalar indices can only speed up scans for basic filters using        equality, comparison, range (e.g. ``my_col BETWEEN 0 AND 100``), and set        membership (e.g. `my_col IN (0, 1, 2)`)        Scalar indices can be used if the filter contains multiple indexed columns and        the filter criteria are AND'd or OR'd together        (e.g. ``my_col < 0 AND other_col> 100``)        Scalar indices may be used if the filter contains non-indexed columns but,        depending on the structure of the filter, they may not be usable.  For example,        if the column ``not_indexed`` does not have a scalar index then the filter        ``my_col = 0 OR not_indexed = 1`` will not be able to use any scalar index on        ``my_col``.        """raiseNotImplementedErrordefcreate_fts_index(self,field_names:Union[str,List[str]],*,ordering_field_names:Optional[Union[str,List[str]]]=None,replace:bool=False,writer_heap_size:Optional[int]=1024*1024*1024,use_tantivy:bool=False,tokenizer_name:Optional[str]=None,with_position:bool=False,# tokenizer configs:base_tokenizer:BaseTokenizerType="simple",language:str="English",max_token_length:Optional[int]=40,lower_case:bool=True,stem:bool=True,remove_stop_words:bool=True,ascii_folding:bool=True,ngram_min_length:int=3,ngram_max_length:int=3,prefix_only:bool=False,wait_timeout:Optional[timedelta]=None,):"""Create a full-text search index on the table.        Warning - this API is highly experimental and is highly likely to change        in the future.        Parameters        ----------        field_names: str or list of str            The name(s) of the field to index.            can be only str if use_tantivy=True for now.        replace: bool, default False            If True, replace the existing index if it exists. Note that this is            not yet an atomic operation; the index will be temporarily            unavailable while the new index is being created.        writer_heap_size: int, default 1GB            Only available with use_tantivy=True        ordering_field_names:            A list of unsigned type fields to index to optionally order            results on at search time.            only available with use_tantivy=True        tokenizer_name: str, default "default"            The tokenizer to use for the index. Can be "raw", "default" or the 2 letter            language code followed by "_stem". So for english it would be "en_stem".            For available languages see: https://docs.rs/tantivy/latest/tantivy/tokenizer/enum.Language.html        use_tantivy: bool, default False            If True, use the legacy full-text search implementation based on tantivy.            If False, use the new full-text search implementation based on lance-index.        with_position: bool, default False            Only available with use_tantivy=False            If False, do not store the positions of the terms in the text.            This can reduce the size of the index and improve indexing speed.            But it will raise an exception for phrase queries.        base_tokenizer : str, default "simple"            The base tokenizer to use for tokenization. Options are:            - "simple": Splits text by whitespace and punctuation.            - "whitespace": Split text by whitespace, but not punctuation.            - "raw": No tokenization. The entire text is treated as a single token.            - "ngram": N-Gram tokenizer.        language : str, default "English"            The language to use for tokenization.        max_token_length : int, default 40            The maximum token length to index. Tokens longer than this length will be            ignored.        lower_case : bool, default True            Whether to convert the token to lower case. This makes queries            case-insensitive.        stem : bool, default True            Whether to stem the token. Stemming reduces words to their root form.            For example, in English "running" and "runs" would both be reduced to "run".        remove_stop_words : bool, default True            Whether to remove stop words. Stop words are common words that are often            removed from text before indexing. For example, in English "the" and "and".        ascii_folding : bool, default True            Whether to fold ASCII characters. This converts accented characters to            their ASCII equivalent. For example, "café" would be converted to "cafe".        ngram_min_length: int, default 3            The minimum length of an n-gram.        ngram_max_length: int, default 3            The maximum length of an n-gram.        prefix_only: bool, default False            Whether to only index the prefix of the token for ngram tokenizer.        wait_timeout: timedelta, optional            The timeout to wait if indexing is asynchronous.        """raiseNotImplementedError@abstractmethoddefadd(self,data:DATA,mode:AddMode="append",on_bad_vectors:OnBadVectorsType="error",fill_value:float=0.0,)->AddResult:"""Add more data to the [Table](Table).        Parameters        ----------        data: DATA            The data to insert into the table. Acceptable types are:            - list-of-dict            - pandas.DataFrame            - pyarrow.Table or pyarrow.RecordBatch        mode: str            The mode to use when writing the data. Valid values are            "append" and "overwrite".        on_bad_vectors: str, default "error"            What to do if any of the vectors are not the same size or contains NaNs.            One of "error", "drop", "fill".        fill_value: float, default 0.            The value to use when filling vectors. Only used if on_bad_vectors="fill".        Returns        -------        AddResult            An object containing the new version number of the table after adding data.        """raiseNotImplementedErrordefmerge_insert(self,on:Union[str,Iterable[str]])->LanceMergeInsertBuilder:"""        Returns a [`LanceMergeInsertBuilder`][lancedb.merge.LanceMergeInsertBuilder]        that can be used to create a "merge insert" operation        This operation can add rows, update rows, and remove rows all in a single        transaction. It is a very generic tool that can be used to create        behaviors like "insert if not exists", "update or insert (i.e. upsert)",        or even replace a portion of existing data with new data (e.g. replace        all data where month="january")        The merge insert operation works by combining new data from a        **source table** with existing data in a **target table** by using a        join.  There are three categories of records.        "Matched" records are records that exist in both the source table and        the target table. "Not matched" records exist only in the source table        (e.g. these are new data) "Not matched by source" records exist only        in the target table (this is old data)        The builder returned by this method can be used to customize what        should happen for each category of data.        Please note that the data may appear to be reordered as part of this        operation.  This is because updated rows will be deleted from the        dataset and then reinserted at the end with the new values.        Parameters        ----------        on: Union[str, Iterable[str]]            A column (or columns) to join on.  This is how records from the            source table and target table are matched.  Typically this is some            kind of key or id column.        Examples        --------        >>> import lancedb        >>> data = pa.table({"a": [2, 1, 3], "b": ["a", "b", "c"]})        >>> db = lancedb.connect("./.lancedb")        >>> table = db.create_table("my_table", data)        >>> new_data = pa.table({"a": [2, 3, 4], "b": ["x", "y", "z"]})        >>> # Perform a "upsert" operation        >>> res = table.merge_insert("a")     \\        ...      .when_matched_update_all()     \\        ...      .when_not_matched_insert_all() \\        ...      .execute(new_data)        >>> res        MergeResult(version=2, num_updated_rows=2, num_inserted_rows=1, num_deleted_rows=0)        >>> # The order of new rows is non-deterministic since we use        >>> # a hash-join as part of this operation and so we sort here        >>> table.to_arrow().sort_by("a").to_pandas()           a  b        0  1  b        1  2  x        2  3  y        3  4  z        """# noqa: E501on=[on]ifisinstance(on,str)elselist(iter(on))returnLanceMergeInsertBuilder(self,on)@abstractmethoddefsearch(self,query:Optional[Union[VEC,str,"PIL.Image.Image",Tuple,FullTextQuery]]=None,vector_column_name:Optional[str]=None,query_type:QueryType="auto",ordering_field_name:Optional[str]=None,fts_columns:Optional[Union[str,List[str]]]=None,)->LanceQueryBuilder:"""Create a search query to find the nearest neighbors        of the given query vector. We currently support [vector search][search]        and [full-text search][experimental-full-text-search].        All query options are defined in        [LanceQueryBuilder][lancedb.query.LanceQueryBuilder].        Examples        --------        >>> import lancedb        >>> db = lancedb.connect("./.lancedb")        >>> data = [        ...    {"original_width": 100, "caption": "bar", "vector": [0.1, 2.3, 4.5]},        ...    {"original_width": 2000, "caption": "foo",  "vector": [0.5, 3.4, 1.3]},        ...    {"original_width": 3000, "caption": "test", "vector": [0.3, 6.2, 2.6]}        ... ]        >>> table = db.create_table("my_table", data)        >>> query = [0.4, 1.4, 2.4]        >>> (table.search(query)        ...     .where("original_width > 1000", prefilter=True)        ...     .select(["caption", "original_width", "vector"])        ...     .limit(2)        ...     .to_pandas())          caption  original_width           vector  _distance        0     foo            2000  [0.5, 3.4, 1.3]   5.220000        1    test            3000  [0.3, 6.2, 2.6]  23.089996        Parameters        ----------        query: list/np.ndarray/str/PIL.Image.Image, default None            The targetted vector to search for.            - *default None*.            Acceptable types are: list, np.ndarray, PIL.Image.Image            - If None then the select/where/limit clauses are applied to filter            the table        vector_column_name: str, optional            The name of the vector column to search.            The vector column needs to be a pyarrow fixed size list type            - If not specified then the vector column is inferred from            the table schema            - If the table has multiple vector columns then the *vector_column_name*            needs to be specified. Otherwise, an error is raised.        query_type: str            *default "auto"*.            Acceptable types are: "vector", "fts", "hybrid", or "auto"            - If "auto" then the query type is inferred from the query;                - If `query` is a list/np.ndarray then the query type is                "vector";                - If `query` is a PIL.Image.Image then either do vector search,                or raise an error if no corresponding embedding function is found.            - If `query` is a string, then the query type is "vector" if the            table has embedding functions else the query type is "fts"        Returns        -------        LanceQueryBuilder            A query builder object representing the query.            Once executed, the query returns            - selected columns            - the vector            - and also the "_distance" column which is the distance between the query            vector and the returned vector.        """raiseNotImplementedError@abstractmethoddef_execute_query(self,query:Query,*,batch_size:Optional[int]=None,timeout:Optional[timedelta]=None,)->pa.RecordBatchReader:...@abstractmethoddef_explain_plan(self,query:Query,verbose:Optional[bool]=False)->str:...@abstractmethoddef_analyze_plan(self,query:Query)->str:...@abstractmethoddef_do_merge(self,merge:LanceMergeInsertBuilder,new_data:DATA,on_bad_vectors:OnBadVectorsType,fill_value:float,)->MergeResult:...@abstractmethoddefdelete(self,where:str)->DeleteResult:"""Delete rows from the table.        This can be used to delete a single row, many rows, all rows, or        sometimes no rows (if your predicate matches nothing).        Parameters        ----------        where: str            The SQL where clause to use when deleting rows.            - For example, 'x = 2' or 'x IN (1, 2, 3)'.            The filter must not be empty, or it will error.        Returns        -------        DeleteResult            An object containing the new version number of the table after deletion.        Examples        --------        >>> import lancedb        >>> data = [        ...    {"x": 1, "vector": [1.0, 2]},        ...    {"x": 2, "vector": [3.0, 4]},        ...    {"x": 3, "vector": [5.0, 6]}        ... ]        >>> db = lancedb.connect("./.lancedb")        >>> table = db.create_table("my_table", data)        >>> table.to_pandas()           x      vector        0  1  [1.0, 2.0]        1  2  [3.0, 4.0]        2  3  [5.0, 6.0]        >>> table.delete("x = 2")        DeleteResult(version=2)        >>> table.to_pandas()           x      vector        0  1  [1.0, 2.0]        1  3  [5.0, 6.0]        If you have a list of values to delete, you can combine them into a        stringified list and use the `IN` operator:        >>> to_remove = [1, 5]        >>> to_remove = ", ".join([str(v) for v in to_remove])        >>> to_remove        '1, 5'        >>> table.delete(f"x IN ({to_remove})")        DeleteResult(version=3)        >>> table.to_pandas()           x      vector        0  3  [5.0, 6.0]        """raiseNotImplementedError@abstractmethoddefupdate(self,where:Optional[str]=None,values:Optional[dict]=None,*,values_sql:Optional[Dict[str,str]]=None,)->UpdateResult:"""        This can be used to update zero to all rows depending on how many        rows match the where clause. If no where clause is provided, then        all rows will be updated.        Either `values` or `values_sql` must be provided. You cannot provide        both.        Parameters        ----------        where: str, optional            The SQL where clause to use when updating rows. For example, 'x = 2'            or 'x IN (1, 2, 3)'. The filter must not be empty, or it will error.        values: dict, optional            The values to update. The keys are the column names and the values            are the values to set.        values_sql: dict, optional            The values to update, expressed as SQL expression strings. These can            reference existing columns. For example, {"x": "x + 1"} will increment            the x column by 1.        Returns        -------        UpdateResult            - rows_updated: The number of rows that were updated            - version: The new version number of the table after the update        Examples        --------        >>> import lancedb        >>> import pandas as pd        >>> data = pd.DataFrame({"x": [1, 2, 3], "vector": [[1.0, 2], [3, 4], [5, 6]]})        >>> db = lancedb.connect("./.lancedb")        >>> table = db.create_table("my_table", data)        >>> table.to_pandas()           x      vector        0  1  [1.0, 2.0]        1  2  [3.0, 4.0]        2  3  [5.0, 6.0]        >>> table.update(where="x = 2", values={"vector": [10.0, 10]})        UpdateResult(rows_updated=1, version=2)        >>> table.to_pandas()           x        vector        0  1    [1.0, 2.0]        1  3    [5.0, 6.0]        2  2  [10.0, 10.0]        >>> table.update(values_sql={"x": "x + 1"})        UpdateResult(rows_updated=3, version=3)        >>> table.to_pandas()           x        vector        0  2    [1.0, 2.0]        1  4    [5.0, 6.0]        2  3  [10.0, 10.0]        """raiseNotImplementedError@abstractmethoddefcleanup_old_versions(self,older_than:Optional[timedelta]=None,*,delete_unverified:bool=False,)->"CleanupStats":"""        Clean up old versions of the table, freeing disk space.        Parameters        ----------        older_than: timedelta, default None            The minimum age of the version to delete. If None, then this defaults            to two weeks.        delete_unverified: bool, default False            Because they may be part of an in-progress transaction, files newer            than 7 days old are not deleted by default. If you are sure that            there are no in-progress transactions, then you can set this to True            to delete all files older than `older_than`.        Returns        -------        CleanupStats            The stats of the cleanup operation, including how many bytes were            freed.        See Also        --------        [Table.optimize][lancedb.table.Table.optimize]: A more comprehensive            optimization operation that includes cleanup as well as other operations.        Notes        -----        This function is not available in LanceDb Cloud (since LanceDB        Cloud manages cleanup for you automatically)        """@abstractmethoddefcompact_files(self,*args,**kwargs):"""        Run the compaction process on the table.        This can be run after making several small appends to optimize the table        for faster reads.        Arguments are passed onto Lance's        [compact_files][lance.dataset.DatasetOptimizer.compact_files].        For most cases, the default should be fine.        See Also        --------        [Table.optimize][lancedb.table.Table.optimize]: A more comprehensive            optimization operation that includes cleanup as well as other operations.        Notes        -----        This function is not available in LanceDB Cloud (since LanceDB        Cloud manages compaction for you automatically)        """@abstractmethoddefoptimize(self,*,cleanup_older_than:Optional[timedelta]=None,delete_unverified:bool=False,retrain:bool=False,):"""        Optimize the on-disk data and indices for better performance.        Modeled after ``VACUUM`` in PostgreSQL.        Optimization covers three operations:         * Compaction: Merges small files into larger ones         * Prune: Removes old versions of the dataset         * Index: Optimizes the indices, adding new data to existing indices        Parameters        ----------        cleanup_older_than: timedelta, optional default 7 days            All files belonging to versions older than this will be removed.  Set            to 0 days to remove all versions except the latest.  The latest version            is never removed.        delete_unverified: bool, default False            Files leftover from a failed transaction may appear to be part of an            in-progress operation (e.g. appending new data) and these files will not            be deleted unless they are at least 7 days old. If delete_unverified is True            then these files will be deleted regardless of their age.        retrain: bool, default False            If True, retrain the vector indices, this would refine the IVF clustering            and quantization, which may improve the search accuracy. It's faster than            re-creating the index from scratch, so it's recommended to try this first,            when the data distribution has changed significantly.        Experimental API        ----------------        The optimization process is undergoing active development and may change.        Our goal with these changes is to improve the performance of optimization and        reduce the complexity.        That being said, it is essential today to run optimize if you want the best        performance.  It should be stable and safe to use in production, but it our        hope that the API may be simplified (or not even need to be called) in the        future.        The frequency an application shoudl call optimize is based on the frequency of        data modifications.  If data is frequently added, deleted, or updated then        optimize should be run frequently.  A good rule of thumb is to run optimize if        you have added or modified 100,000 or more records or run more than 20 data        modification operations.        """@abstractmethoddeflist_indices(self)->Iterable[IndexConfig]:"""        List all indices that have been created with        [Table.create_index][lancedb.table.Table.create_index]        """@abstractmethoddefindex_stats(self,index_name:str)->Optional[IndexStatistics]:"""        Retrieve statistics about an index        Parameters        ----------        index_name: str            The name of the index to retrieve statistics for        Returns        -------        IndexStatistics or None            The statistics about the index. Returns None if the index does not exist.        """@abstractmethoddefadd_columns(self,transforms:Dict[str,str]|pa.Field|List[pa.Field]|pa.Schema):"""        Add new columns with defined values.        Parameters        ----------        transforms: Dict[str, str], pa.Field, List[pa.Field], pa.Schema            A map of column name to a SQL expression to use to calculate the            value of the new column. These expressions will be evaluated for            each row in the table, and can reference existing columns.            Alternatively, a pyarrow Field or Schema can be provided to add            new columns with the specified data types. The new columns will            be initialized with null values.        Returns        -------        AddColumnsResult            version: the new version number of the table after adding columns.        """@abstractmethoddefalter_columns(self,*alterations:Iterable[Dict[str,str]]):"""        Alter column names and nullability.        Parameters        ----------        alterations : Iterable[Dict[str, Any]]            A sequence of dictionaries, each with the following keys:            - "path": str                The column path to alter. For a top-level column, this is the name.                For a nested column, this is the dot-separated path, e.g. "a.b.c".            - "rename": str, optional                The new name of the column. If not specified, the column name is                not changed.            - "data_type": pyarrow.DataType, optional               The new data type of the column. Existing values will be casted               to this type. If not specified, the column data type is not changed.            - "nullable": bool, optional                Whether the column should be nullable. If not specified, the column                nullability is not changed. Only non-nullable columns can be changed                to nullable. Currently, you cannot change a nullable column to                non-nullable.        Returns        -------        AlterColumnsResult            version: the new version number of the table after the alteration.        """@abstractmethoddefdrop_columns(self,columns:Iterable[str])->DropColumnsResult:"""        Drop columns from the table.        Parameters        ----------        columns : Iterable[str]            The names of the columns to drop.        Returns        -------        DropColumnsResult            version: the new version number of the table dropping the columns.        """@abstractmethoddefcheckout(self,version:Union[int,str]):"""        Checks out a specific version of the Table        Any read operation on the table will now access the data at the checked out        version. As a consequence, calling this method will disable any read consistency        interval that was previously set.        This is a read-only operation that turns the table into a sort of "view"        or "detached head".  Other table instances will not be affected.  To make the        change permanent you can use the `[Self::restore]` method.        Any operation that modifies the table will fail while the table is in a checked        out state.        Parameters        ----------        version: int | str,            The version to check out. A version number (`int`) or a tag            (`str`) can be provided.        To return the table to a normal state use `[Self::checkout_latest]`        """@abstractmethoddefcheckout_latest(self):"""        Ensures the table is pointing at the latest version        This can be used to manually update a table when the read_consistency_interval        is None        It can also be used to undo a `[Self::checkout]` operation        """@abstractmethoddefrestore(self,version:Optional[Union[int,str]]=None):"""Restore a version of the table. This is an in-place operation.        This creates a new version where the data is equivalent to the        specified previous version. Data is not copied (as of python-v0.2.1).        Parameters        ----------        version : int or str, default None            The version number or version tag to restore.            If unspecified then restores the currently checked out version.            If the currently checked out version is the            latest version then this is a no-op.        """@abstractmethoddeflist_versions(self)->List[Dict[str,Any]]:"""List all versions of the table"""@cached_propertydef_dataset_uri(self)->str:return_table_uri(self._conn.uri,self.name)def_get_fts_index_path(self)->Tuple[str,pa_fs.FileSystem,bool]:from.remote.tableimportRemoteTableifisinstance(self,RemoteTable)orget_uri_scheme(self._dataset_uri)!="file":return("",None,False)path=join_uri(self._dataset_uri,"_indices","fts")fs,path=fs_from_uri(path)index_exists=fs.get_file_info(path).type!=pa_fs.FileType.NotFoundreturn(path,fs,index_exists)@abstractmethoddefuses_v2_manifest_paths(self)->bool:"""        Check if the table is using the new v2 manifest paths.        Returns        -------        bool            True if the table is using the new v2 manifest paths, False otherwise.        """@abstractmethoddefmigrate_v2_manifest_paths(self):"""        Migrate the manifest paths to the new format.        This will update the manifest to use the new v2 format for paths.        This function is idempotent, and can be run multiple times without        changing the state of the object store.        !!! danger            This should not be run while other concurrent operations are happening.            And it should also run until completion before resuming other operations.        You can use        [Table.uses_v2_manifest_paths][lancedb.table.Table.uses_v2_manifest_paths]        to check if the table is already using the new path style.        """