Usage

1. git clone https://github.com/XuefengHuang/lianjia-scrawler.git2. cd lianjia-scrawler# If you'd like not to use [virtualenv](https://virtualenv.pypa.io/en/stable/), please skip step 3 and 4.3. virtualenv lianjia4. source lianjia/bin/activate5. pip install -r requirements.txt

[Mysql]enable = Truescheme = testhost = 127.0.0.1port = 3306user = rootpassword = secret[Sqlite]enable = Falsedbname = lianjia.db[Postgresql]enable = Falsescheme = testhost = 127.0.0.1user = postgrespassword = secret

Database Information

# All communities in every region. class Community(BaseModel):id = PrimaryKeyField()title = CharField()link = CharField(unique=True)district = CharField()bizcircle = CharField()tagList = CharField()# All onsale house information in every community.class Houseinfo(BaseModel):houseID = BigIntegerField(primary_key=True)title = CharField()link = CharField()community = CharField()years = CharField()housetype = CharField()square = CharField()direction = CharField()floor = CharField()taxtype = CharField()totalPrice = IntegerField()unitPrice = IntegerField()followInfo = CharField()validdate = DateTimeField(default=datetime.datetime.now)# All onsale house history price in every community.class Hisprice(BaseModel):houseID = BigIntegerField()totalPrice = IntegerField()date = DateTimeField(default=datetime.datetime.now)class Meta:primary_key = CompositeKey('houseID', 'totalPrice')# All sold house information in every community.class Sellinfo(BaseModel):houseID = BigIntegerField(primary_key=True)title = CharField()link = CharField()community = CharField()years = CharField()housetype = CharField()square = CharField()direction = CharField()floor = CharField()status = CharField()source = CharField()totalPrice = IntegerField()unitPrice = IntegerField()dealdate = DateField()updatedate = DateTimeField(default=datetime.datetime.now)# All rent house information in every community.class Rentinfo(BaseModel):houseID = BigIntegerField(primary_key=True)title = CharField()link = CharField()region = CharField()zone = CharField()meters = CharField()other = CharField()subway = CharField()decoration = CharField()heating = CharField()price = IntegerField()pricepre = CharField()updatedate = DateTimeField(default=datetime.datetime.now)

Examples: