Skip to content
This repository was archived by the owner on May 5, 2023. It is now read-only.

Commit ebf9d12

Browse files
leafcoderleafcoder
committed
improvement: update models and spider for saving less data
1 parent cb76e95 commit ebf9d12

File tree

8 files changed

+182
-493
lines changed

8 files changed

+182
-493
lines changed

‎ncovapi/admin.py

Lines changed: 1 addition & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -18,67 +18,11 @@ class BaseAdmin(admin.ModelAdmin):
1818
preserve_filters = True
1919

2020

21-
@admin.register(models.Crawler)
22-
class CrawlerAdmin(BaseAdmin):
23-
24-
list_display = (
25-
'id', 'crawlTime', 'createTime', 'modifyTime'
26-
)
27-
28-
2921
@admin.register(models.Statistics)
3022
class StatisticsAdmin(BaseAdmin):
3123

32-
list_display = (
33-
'id', 'countryType', 'seriousCount', 'currentConfirmedCount',
34-
'confirmedCount', 'suspectedCount', 'curedCount', 'deadCount'
35-
)
36-
search_fields = ('crawler_id', )
37-
list_filter = ('countryType', )
38-
39-
40-
@admin.register(models.Notice)
41-
class NoticeAdmin(BaseAdmin):
42-
43-
list_display = (
44-
'id', 'crawler', 'remarks', 'notes', 'generalRemark'
45-
)
46-
47-
@admin.register(models.WHOArticle)
48-
class WHOArticleAdmin(BaseAdmin):
24+
pass
4925

50-
list_display = ('id', 'title', 'linkUrl', 'imgUrl')
51-
52-
@admin.register(models.Recommend)
53-
class RecommendAdmin(BaseAdmin):
54-
55-
list_display = (
56-
'id', 'title', 'linkUrl', 'imgUrl', 'contentType',
57-
'countryType', 'recordStatus')
58-
59-
@admin.register(models.Timeline)
60-
class TimelineAdmin(BaseAdmin):
61-
62-
list_display = (
63-
'id', 'title', 'summary', 'pubDateStr', 'infoSource', 'sourceUrl')
64-
65-
@admin.register(models.Wiki)
66-
class WikiAdmin(BaseAdmin):
67-
68-
list_display = ('id', 'title', 'linkUrl', 'imgUrl', 'description')
69-
70-
@admin.register(models.GoodsGuide)
71-
class GoodsGuideAdmin(BaseAdmin):
72-
73-
list_display = (
74-
'categoryName', 'title', 'recordStatus', 'contentImgUrls')
75-
76-
@admin.register(models.Rumor)
77-
class RumorAdmin(BaseAdmin):
78-
79-
list_display = (
80-
'title', 'mainSummary', 'summary', 'body',
81-
'sourceUrl', 'score', 'rumorType')
8226

8327
@admin.register(models.City)
8428
class CityAdmin(BaseAdmin):

‎ncovapi/filters.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,21 @@ class CityFilter(django_filters.rest_framework.FilterSet):
1111

1212
provinceShortNames = CharInFilter(
1313
field_name='province__provinceShortName', lookup_expr='in')
14+
provinceNames = CharInFilter(
15+
field_name='province__provinceName', lookup_expr='in')
1416
cityNames = CharInFilter(
1517
field_name='cityName', lookup_expr='in')
18+
19+
provinceShortName = django_filters.CharFilter(
20+
field_name='province__provinceShortName', lookup_expr='exact')
21+
provinceName = django_filters.CharFilter(
22+
field_name='province__provinceName', lookup_expr='exact')
23+
cityName = django_filters.CharFilter(
24+
field_name='cityName', lookup_expr='exact')
1625

1726
class Meta:
1827
model = City
19-
fields = ['provinceShortNames', 'cityName']
28+
fields = ['provinceShortName', 'provinceName', 'cityName']
2029

2130

2231
class ProvinceFilter(django_filters.rest_framework.FilterSet):
@@ -26,6 +35,11 @@ class ProvinceFilter(django_filters.rest_framework.FilterSet):
2635
provinceNames = CharInFilter(
2736
field_name='provinceName', lookup_expr='in')
2837

38+
provinceShortName = django_filters.CharFilter(
39+
field_name='provinceShortName', lookup_expr='exact')
40+
provinceName = django_filters.CharFilter(
41+
field_name='provinceName', lookup_expr='exact')
42+
2943
class Meta:
3044
model = Province
3145
fields = ['provinceName', 'provinceShortName']

‎ncovapi/models.py

Lines changed: 21 additions & 185 deletions
Original file line numberDiff line numberDiff line change
@@ -1,192 +1,39 @@
11
from django.db import models
22
from django.utils import timezone
3-
from django_mysql.models import ListCharField
3+
from django_mysql.models import JSONField
44

55

6-
class Crawler(models.Model):
6+
class Statistics(models.Model):
7+
78

9+
JSON_FIELDS = (
10+
'globalStatistics', 'domesticStatistics', 'internationalStatistics',
11+
'remarks', 'notes', 'WHOArticle', 'recommends', 'timelines',
12+
'wikis', 'goodsGuides', 'rumors'
13+
)
14+
15+
globalStatistics = models.TextField(default='{}')
16+
domesticStatistics = models.TextField(default='{}')
17+
internationalStatistics = models.TextField(default='{}')
18+
remarks = models.TextField(default='[]')
19+
notes = models.TextField(default='[]')
20+
generalRemark = models.TextField(default='')
21+
WHOArticle = models.TextField(verbose_name='WHO 文章', default='{}')
22+
recommends = models.TextField(verbose_name='防护知识', default='[]')
23+
timelines = models.TextField(verbose_name='时间线事件', default='[]')
24+
wikis = models.TextField(verbose_name='Wiki', default='[]')
25+
goodsGuides = models.TextField(verbose_name='购物指南', default='[]')
26+
rumors = models.TextField(verbose_name='辟谣与防护', default='[]')
827
modifyTime = models.DateTimeField(null=True)
928
createTime = models.DateTimeField(null=True)
1029
crawlTime = models.DateTimeField(
1130
"抓取时间", default=timezone.now, editable=False)
1231

13-
class Meta:
14-
verbose_name = "抓取版本"
15-
verbose_name_plural = "抓取版本"
16-
17-
18-
class Statistics(models.Model):
19-
20-
GLOBAL = 0
21-
DOMESTIC = 1
22-
INTERNATIONAL = 2
23-
COUNTRY_TYPES = (
24-
(GLOBAL, '全球'),
25-
(DOMESTIC, '国内'),
26-
(INTERNATIONAL, '国外')
27-
)
28-
29-
countryType = models.IntegerField(choices=COUNTRY_TYPES)
30-
currentConfirmedCount = models.IntegerField(default=0)
31-
confirmedCount = models.IntegerField(default=0)
32-
suspectedCount = models.IntegerField(default=0)
33-
seriousCount = models.IntegerField('现存无症状', default=0)
34-
curedCount = models.IntegerField(default=0)
35-
deadCount = models.IntegerField(default=0)
36-
currentConfirmedIncr = models.IntegerField(default=0)
37-
confirmedIncr = models.IntegerField(default=0)
38-
suspectedIncr = models.IntegerField(default=0)
39-
curedIncr = models.IntegerField(default=0)
40-
deadIncr = models.IntegerField(default=0)
41-
crawler = models.ForeignKey(
42-
"Crawler", on_delete=models.CASCADE, related_name="statistics",
43-
db_column="crawlerId"
44-
)
45-
4632
class Meta:
4733
verbose_name = '统计数据'
4834
verbose_name_plural = '统计数据'
4935

5036

51-
class Notice(models.Model):
52-
53-
remarks = ListCharField(
54-
models.CharField(max_length=100), size=10, max_length=100*11)
55-
notes = ListCharField(
56-
models.CharField(max_length=100), size=10, max_length=100*11)
57-
generalRemark = models.TextField(null=True)
58-
crawler = models.ForeignKey(
59-
"Crawler", on_delete=models.CASCADE, related_name="notices",
60-
db_column="crawlerId"
61-
)
62-
63-
class Meta:
64-
verbose_name = '注意信息'
65-
verbose_name_plural = '注意信息'
66-
67-
68-
class WHOArticle(models.Model):
69-
70-
title = models.CharField(max_length=100)
71-
linkUrl = models.URLField(max_length=200)
72-
imgUrl = models.URLField(max_length=200)
73-
crawler = models.OneToOneField(
74-
"Crawler", on_delete=models.CASCADE, related_name="WHO_article",
75-
db_column="crawlerId"
76-
)
77-
78-
class Meta:
79-
verbose_name = 'WHO 文章'
80-
verbose_name_plural = 'WHO 文章'
81-
82-
83-
class Recommend(models.Model):
84-
85-
CONTENT_TYPES = [
86-
(1, '我要出行'),
87-
(2, '家有小孩'),
88-
(3, '未知'),
89-
(4, '我宅在家'),
90-
(5, '未知'),
91-
(6, '未知'),
92-
(7, '未知')
93-
]
94-
GLOBAL = 0
95-
DOMESTIC = 1
96-
INTERNATIONAL = 2
97-
COUNTRY_TYPES = (
98-
(GLOBAL, '全球'),
99-
(DOMESTIC, '国内'),
100-
(INTERNATIONAL, '国外')
101-
)
102-
103-
title = models.CharField(max_length=100)
104-
linkUrl = models.URLField(max_length=200)
105-
imgUrl = models.URLField(max_length=200)
106-
contentType = models.IntegerField(choices=CONTENT_TYPES)
107-
recordStatus = models.IntegerField()
108-
countryType = models.IntegerField(choices=COUNTRY_TYPES)
109-
sort = models.IntegerField()
110-
crawler = models.ForeignKey(
111-
"Crawler", on_delete=models.CASCADE, related_name="recommends",
112-
db_column="crawlerId"
113-
)
114-
115-
class Meta:
116-
verbose_name = '防护知识'
117-
verbose_name_plural = '防护知识'
118-
119-
120-
class Timeline(models.Model):
121-
122-
pubDate = models.IntegerField()
123-
pubDateStr = models.CharField(max_length=50)
124-
title = models.CharField(max_length=100)
125-
summary = models.TextField()
126-
infoSource = models.CharField(max_length=50)
127-
sourceUrl = models.URLField(max_length=200)
128-
crawler = models.ForeignKey(
129-
"Crawler", on_delete=models.CASCADE, related_name="timelines",
130-
db_column="crawlerId"
131-
)
132-
133-
class Meta:
134-
verbose_name = '时间线事件'
135-
verbose_name_plural = '时间线事件'
136-
137-
138-
class Wiki(models.Model):
139-
140-
title = models.CharField(max_length=100)
141-
linkUrl = models.URLField(max_length=200)
142-
imgUrl = models.URLField(max_length=200)
143-
description = models.TextField()
144-
crawler = models.ForeignKey(
145-
"Crawler", on_delete=models.CASCADE, related_name="wikis",
146-
db_column="crawlerId"
147-
)
148-
149-
class Meta:
150-
verbose_name = 'Wiki'
151-
verbose_name_plural = 'Wiki'
152-
153-
154-
class GoodsGuide(models.Model):
155-
156-
title = models.CharField(max_length=100)
157-
categoryName = models.CharField(max_length=50)
158-
recordStatus = models.IntegerField()
159-
contentImgUrls = ListCharField(
160-
models.CharField(max_length=100), size=10, max_length=100*11)
161-
crawler = models.ForeignKey(
162-
"Crawler", on_delete=models.CASCADE, related_name="goods_guides",
163-
db_column="crawlerId"
164-
)
165-
166-
class Meta:
167-
verbose_name = '购物指南'
168-
verbose_name_plural = '购物指南'
169-
170-
171-
class Rumor(models.Model):
172-
173-
title = models.CharField(max_length=100)
174-
mainSummary = models.TextField()
175-
summary = models.TextField()
176-
body = models.TextField()
177-
sourceUrl = models.URLField(max_length=200)
178-
score = models.IntegerField()
179-
rumorType = models.IntegerField()
180-
crawler = models.ForeignKey(
181-
"Crawler", on_delete=models.CASCADE, related_name="rumors",
182-
db_column="crawlerId"
183-
)
184-
185-
class Meta:
186-
verbose_name = '辟谣与防护'
187-
verbose_name_plural = '辟谣与防护'
188-
189-
19037
class Province(models.Model):
19138

19239
locationId = models.IntegerField()
@@ -204,10 +51,6 @@ class Province(models.Model):
20451
'创建时间', auto_now_add=True, editable=False)
20552
updated = models.DateTimeField(
20653
'更新时间', auto_now=True, editable=False)
207-
crawler = models.ForeignKey(
208-
"Crawler", on_delete=models.CASCADE, related_name="provinces",
209-
db_column="crawlerId"
210-
)
21154

21255
class Meta:
21356
verbose_name = '国内省份'
@@ -231,9 +74,6 @@ class City(models.Model):
23174
"Province", on_delete=models.CASCADE, related_name="cities",
23275
db_column="provinceId"
23376
)
234-
crawler = models.ForeignKey(
235-
"Crawler", on_delete=models.CASCADE, db_column="crawlerId"
236-
)
23777

23878
@property
23979
def provinceName(self):
@@ -273,10 +113,6 @@ class Country(models.Model):
273113
'创建时间', auto_now_add=True, editable=False)
274114
updated = models.DateTimeField(
275115
'更新时间', auto_now=True, editable=False)
276-
crawler = models.ForeignKey(
277-
"Crawler", on_delete=models.CASCADE, related_name="countries",
278-
db_column="countryId"
279-
)
280116

281117
class Meta:
282118
verbose_name = "国家地区"

0 commit comments

Comments
 (0)