@inproceedings{PVS:SPSAL2007,  editor    = {P.V.S., Avinesh and Gali, Karthik},  title     = {Part of Speech Tagging and Chunking using Conditional Random Fields and Transformation Based Learning}  booktitle = {Proceedings of the  Shallow Parsing for South Asian Languages (SPSAL) Workshop, held at IJCAI-07, Hyderabad, India},  series    = {{SPSAL} Workshop Proceedings},  month     = {January},  year      = {2007},  pages     = {21--24},}

Training Data Statistics and System Performances (F1 macro)

Training Data Statistics and System Performances (F1 macro) for NER

Install using Anaconda

    # INSTALL python environment    conda create -n tagger3.6 anaconda python=3.6    source activate tagger3.6        # Install the tokenizer    cd polyglot-tokenizer    python setup.py install        # Install requirements    pip install -r requirements.txt

Run

    python pipeline.py -p predict -l te -t pos -m crf -f txt -e utf -i input_file -o output_file    -l, --languages       select language (2 letter ISO-639 code)                           {hi, be, ml, pu, te, ta, ka, mr, ur}    -t, --tag_type      pos, chunk, parse, ner    -m, --model_type    crf, hmm, lstm    -f, --data_format   ssf, txt, conll    -e, --encoding      utf8, wx   (default: utf8)    -i, --input_file      <input-file>    -o, --output_file     <output-file>    -s, --sent_split      True/False (default: True)    python pipeline.py --help

    # CRF model    python pipeline.py -p train -o outputs -l te -t pos -m crf -e utf -f ssf        # BI-LSTM-CRF model    python pipeline.py -p train -t pos -f conll -m lstm -e utf -l te

    # CRF models     python pipeline.py -p predict -l te -t pos -m crf -f txt -e utf -i data/test/te/test.utf.txt        # BI-LSTM-CRF models    python pipeline.py -p predict -l te -t pos -m lstm -f txt -e utf -i data/test/te/test.utf.txt        # SpaCy models    python spacy_tagger_test.py -l te -t pos

    # CRF model    python pipeline.py -p train -o outputs -l te -t ner -m crf -e utf -f conll        # BI-LSTM-CRF model    python pipeline.py -p train -t ner -f conll -m lstm -e utf -l te

    # CRF model    python pipeline.py -p predict -l hi -t ner -m crf -f txt -e utf -i data/test/hi/test.utf.txt        # BI-LSTM-CRF model    python pipeline.py -p predict -l hi -t ner -m lstm -f txt -e utf -i data/test/hi/test.utf.txt

ToDo List