🔗 Acknowledgements

📜 License

🚀 Quick Start

Building data-flow from Source

docker build -t dataflow . -f Dockerfiledocker buildx build --provenance false --platform linux/amd64 -t dataflow . -f Dockerfiledocker buildx build --provenance false --platform linux/arm64 -t dataflow . -f Dockerfile

Prerequisites

docker run -d --name dataflow-pg \   -p 5433:5432 \   -v /tmp/data_flow/pgdata:/var/lib/postgresql/data \   -e POSTGRES_DB=data_flow \   -e POSTGRES_USER=postgres \   -e POSTGRES_PASSWORD=postgres \   opencsg-registry.cn-beijing.cr.aliyuncs.com/opencsghq/csghub/postgres:15.10

docker run -d --name dataflow-mongo \   -p 27017:27017 \   -v /tmp/data_flow/mongodata:/data/db \   -e MONGO_INITDB_ROOT_USERNAME=root \   -e MONGO_INITDB_ROOT_PASSWORD=example \   opencsg-registry.cn-beijing.cr.aliyuncs.com/opencsghq/mongo:8.0.12

docker run -d --name dataflow-redis \   -p 16379:6379 \   -v /tmp/data_flow/redisdata:/data \   opencsg-registry.cn-beijing.cr.aliyuncs.com/opencsghq/redis:7.2.5

Installation data-flow

docker run -d --name dataflow-api -p 8000:8000 \   -v /tmp/data_flow/apidata:/data/dataflow_data \   -c"uvicorn data_server.main:app --host 0.0.0.0 --port 8000" \   -e DATA_DIR=/data/dataflow_data \   -e CSGHUB_ENDPOINT=https://hub.opencsg.com \   -e MAX_WORKERS=99 \   -e RAY_ADDRESS=auto \   -e RAY_ENABLE=False \   -e RAY_LOG_DIR=/data/ray_output \   -e API_SERVER=0.0.0.0 \   -e API_PORT=8000 \   -e ENABLE_OPENTELEMETRY=False \   -e DATABASE_DB=data_flow \   -e DATABASE_USERNAME=postgres \   -e DATABASE_PASSWORD=postgres \   -e DATABASE_HOSTNAME=127.0.0.1 \   -e DATABASE_PORT=5433 \   -e STUDIO_JUMP_URL=https://data-label.opencsg.com \   -e REDIS_HOST_URL=redis://127.0.0.1:16379 \   -e MONG_HOST_URL=mongodb://root:example@127.0.0.1:27017 \   dataflow

Installation data-flow-celery

docker run -d --name celery-work -p 8001:8001 \   -v /tmp/data_flow/celery-data:/data/dataflow_celery \   -c"celery -A data_celery.main:celery_app worker --loglevel=info --pool=gevent" \   -e DATA_DIR=/data/dataflow_celery \   -e CSGHUB_ENDPOINT=https://hub.opencsg.com \   -e MAX_WORKERS=99 \   -e RAY_ADDRESS=auto \   -e RAY_ENABLE=False \   -e RAY_LOG_DIR=/data/ray_output \   -e API_SERVER=0.0.0.0 \   -e API_PORT=8001 \   -e ENABLE_OPENTELEMETRY=False \   -e DATABASE_DB=data_flow \   -e DATABASE_USERNAME=postgres \   -e DATABASE_PASSWORD=postgres \   -e DATABASE_HOSTNAME=127.0.0.1 \   -e DATABASE_PORT=5433 \   -e REDIS_HOST_URL=redis://127.0.0.1:16379 \   -e MONG_HOST_URL=mongodb://root:example@127.0.0.1:27017 \   dataflow-celery

Run data-flow server in development mode locally

Create a Virtual Environment

uv venv --python 3.10source .venv/bin/activate# orconda create -n  dataflow python=3.10

# Install dependencies#pip install '.[dist]' -i https://pypi.tuna.tsinghua.edu.cn/simple/#pip install '.[tools]' -i https://pypi.tuna.tsinghua.edu.cn/simple/#pip install '.[sci]' -i https://pypi.tuna.tsinghua.edu.cn/simple/#pip install -r docker/requirements.txtuv pip install -r docker/dataflow_requirements.txt -i https://mirrors.aliyun.com/pypi/simple/# Run the server locallyuvicorn data_server.main:app --reload

Run data-flow-celery server in development mode locally

# Run the celery server locallycelery -A data_celery.main:celery_app worker --loglevel=info --pool=gevent

🛣️ Roadmap

🤝 Contributing

📞 Contact