Skip to content

Commit 081b043

Browse files
author
Hooram Nam
committed
cahce invalidation
1 parent 3872db1 commit 081b043

File tree

4 files changed

+106
-36
lines changed

4 files changed

+106
-36
lines changed

‎api/directory_watcher.py

+80-14
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,14 @@ def scan_photos():
3535
for image_path in tqdm(image_paths):
3636
if image_path.lower().endswith('.jpg'):
3737
try:
38-
img_abs_path = os.path.abspath(os.path.join(image_dir,image_path))
39-
try:
40-
hash_md5 = hashlib.md5()
41-
with open(img_abs_path, "rb") as f:
42-
for chunk in iter(lambda: f.read(4096), b""):
43-
hash_md5.update(chunk)
44-
image_hash = hash_md5.hexdigest()
45-
qs = Photo.objects.filter(image_hash=image_hash)
46-
except:
47-
raise Exception
38+
img_abs_path = image_path
39+
hash_md5 = hashlib.md5()
40+
with open(img_abs_path, "rb") as f:
41+
for chunk in iter(lambda: f.read(4096), b""):
42+
hash_md5.update(chunk)
43+
image_hash = hash_md5.hexdigest()
44+
qs = Photo.objects.filter(image_hash=image_hash)
45+
4846
if qs.count() < 1:
4947
photo = Photo(image_path=img_abs_path)
5048
photo.added_on = datetime.datetime.now()
@@ -54,18 +52,18 @@ def scan_photos():
5452
start = datetime.datetime.now()
5553
photo._generate_thumbnail()
5654
elapsed = (datetime.datetime.now() - start).total_seconds()
57-
# print('thumbnail get', elapsed)
55+
print('thumbnail get', elapsed)
5856

5957
start = datetime.datetime.now()
6058
photo._save_image_to_db()
6159
elapsed = (datetime.datetime.now() - start).total_seconds()
62-
# print('image save', elapsed)
60+
print('image save', elapsed)
6361

6462
start = datetime.datetime.now()
6563
photo._extract_exif()
6664
photo.save()
6765
elapsed = (datetime.datetime.now() - start).total_seconds()
68-
# print('exif extraction', elapsed)
66+
print('exif extraction', elapsed)
6967

7068
# start = datetime.datetime.now()
7169
# photo._geolocate()
@@ -76,7 +74,7 @@ def scan_photos():
7674
start = datetime.datetime.now()
7775
photo._extract_faces()
7876
elapsed = (datetime.datetime.now() - start).total_seconds()
79-
# print('face extraction', elapsed)
77+
print('face extraction', elapsed)
8078

8179
start = datetime.datetime.now()
8280
photo._add_to_album_date()
@@ -87,6 +85,10 @@ def scan_photos():
8785
print("photo already exists in db")
8886
except Exception as e:
8987
print("could not load image %s"%image_path)
88+
try:
89+
print(e.message)
90+
except:
91+
pass
9092

9193
return {"new_photo_count":added_photo_count, "status":True}
9294
# photos = Photo.objects.all()
@@ -96,3 +98,67 @@ def scan_photos():
9698
# print(photo.face_set.all())
9799

98100

101+
if __name__ == "__main__":
102+
image_paths = []
103+
for image_dir in image_dirs:
104+
image_paths.extend([os.path.join(dp, f) for dp, dn, fn in os.walk(image_dir) for f in fn]
105+
)
106+
107+
added_photo_count = 0
108+
for image_path in tqdm(image_paths):
109+
if image_path.lower().endswith('.jpg'):
110+
try:
111+
img_abs_path = image_path
112+
hash_md5 = hashlib.md5()
113+
with open(img_abs_path, "rb") as f:
114+
for chunk in iter(lambda: f.read(4096), b""):
115+
hash_md5.update(chunk)
116+
image_hash = hash_md5.hexdigest()
117+
qs = Photo.objects.filter(image_hash=image_hash)
118+
119+
if qs.count() < 1:
120+
photo = Photo(image_path=img_abs_path)
121+
photo.added_on = datetime.datetime.now()
122+
photo.save()
123+
photo._generate_md5()
124+
125+
start = datetime.datetime.now()
126+
photo._generate_thumbnail()
127+
elapsed = (datetime.datetime.now() - start).total_seconds()
128+
print('thumbnail get', elapsed)
129+
130+
start = datetime.datetime.now()
131+
photo._save_image_to_db()
132+
elapsed = (datetime.datetime.now() - start).total_seconds()
133+
print('image save', elapsed)
134+
135+
start = datetime.datetime.now()
136+
photo._extract_exif()
137+
photo.save()
138+
elapsed = (datetime.datetime.now() - start).total_seconds()
139+
print('exif extraction', elapsed)
140+
141+
# start = datetime.datetime.now()
142+
# photo._geolocate()
143+
# photo.save()
144+
# elapsed = (datetime.datetime.now() - start).total_seconds()
145+
# print('geolocation', elapsed)
146+
147+
start = datetime.datetime.now()
148+
photo._extract_faces()
149+
elapsed = (datetime.datetime.now() - start).total_seconds()
150+
print('face extraction', elapsed)
151+
152+
start = datetime.datetime.now()
153+
photo._add_to_album_date()
154+
elapsed = (datetime.datetime.now() - start).total_seconds()
155+
added_photo_count += 1
156+
print(img_abs_path)
157+
else:
158+
print("photo already exists in db")
159+
except Exception as e:
160+
print("could not load image %s"%image_path)
161+
try:
162+
print(e.message)
163+
except:
164+
pass

‎api/models.py

+13
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,17 @@
1919

2020
from geopy.geocoders import Nominatim
2121

22+
from django.db.models.signals import post_save, post_delete
23+
from django.core.cache import cache
24+
25+
2226
geolocator = Nominatim()
2327
default_tz = pytz.timezone('Asia/Seoul')
2428

29+
30+
def change_api_updated_at(sender=None, instance=None, *args, **kwargs):
31+
cache.set('api_updated_at_timestamp', datetime.utcnow())
32+
2533
def get_album_date(date):
2634
return AlbumDate.objects.get_or_create(date=date)
2735

@@ -266,3 +274,8 @@ class AlbumUser(models.Model):
266274
gps_lat = models.FloatField(blank=True,null=True)
267275
gps_lon = models.FloatField(blank=True,null=True)
268276
photos = models.ManyToManyField(Photo)
277+
278+
279+
for model in [Photo, Person, Face, AlbumDate, AlbumAuto, AlbumUser]:
280+
post_save.connect(receiver=change_api_updated_at, sender=model)
281+
post_delete.connect(receiver=change_api_updated_at, sender=model)

‎api/views.py

+12-17
Original file line numberDiff line numberDiff line change
@@ -62,16 +62,11 @@ class CustomListKeyConstructor(DefaultKeyConstructor):
6262

6363

6464

65-
66-
6765
class StandardResultsSetPagination(PageNumberPagination):
6866
page_size = 1000
6967
page_size_query_param = 'page_size'
7068
max_page_size = 10000
7169

72-
73-
74-
7570
# Create your views here.
7671

7772
class PhotoViewSet(viewsets.ModelViewSet):
@@ -136,13 +131,13 @@ class PersonViewSet(viewsets.ModelViewSet):
136131
serializer_class = PersonSerializer
137132
pagination_class = StandardResultsSetPagination
138133

139-
# @cache_response(key_func=CustomObjectKeyConstructor())
140-
# def retrieve(self, *args, **kwargs):
141-
# return super(PersonViewSet, self).retrieve(*args, **kwargs)
134+
@cache_response(key_func=CustomObjectKeyConstructor())
135+
def retrieve(self, *args, **kwargs):
136+
return super(PersonViewSet, self).retrieve(*args, **kwargs)
142137

143-
# @cache_response(key_func=CustomListKeyConstructor())
144-
# def list(self, *args, **kwargs):
145-
# return super(PersonViewSet, self).list(*args, **kwargs)
138+
@cache_response(key_func=CustomListKeyConstructor())
139+
def list(self, *args, **kwargs):
140+
return super(PersonViewSet, self).list(*args, **kwargs)
146141

147142

148143
class AlbumAutoViewSet(viewsets.ModelViewSet):
@@ -165,13 +160,13 @@ class AlbumPersonViewSet(viewsets.ModelViewSet):
165160
serializer_class = AlbumPersonSerializer
166161
pagination_class = StandardResultsSetPagination
167162

168-
# @cache_response(key_func=CustomObjectKeyConstructor())
169-
# def retrieve(self, *args, **kwargs):
170-
# return super(AlbumPersonViewSet, self).retrieve(*args, **kwargs)
163+
@cache_response(key_func=CustomObjectKeyConstructor())
164+
def retrieve(self, *args, **kwargs):
165+
return super(AlbumPersonViewSet, self).retrieve(*args, **kwargs)
171166

172-
# @cache_response(key_func=CustomListKeyConstructor())
173-
# def list(self, *args, **kwargs):
174-
# return super(AlbumPersonViewSet, self).list(*args, **kwargs)
167+
@cache_response(key_func=CustomListKeyConstructor())
168+
def list(self, *args, **kwargs):
169+
return super(AlbumPersonViewSet, self).list(*args, **kwargs)
175170

176171

177172

‎config.py

+1-5
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,5 @@
33

44
image_dirs = [
55
# '/home/rammi/Downloads/tuebingen',
6-
'/home/hooram/Nextcloud/Photos/korea',
7-
'/home/hooram/Nextcloud/Photos/tuebingen',
8-
# '/Users/hooram/Downloads',
9-
# '/home/hooram/Nextcloud/Camera Uploads',
10-
'/media/hooram/9832141a-0970-48e6-8af7-ea42fa520c17/pictures/DCIM/Camera/',
6+
'/mnt/ext/google_takeout'
117
]

0 commit comments

Comments
 (0)