From d06f1d0f97289707fb0f9ba561f03dc32e1c3327 Mon Sep 17 00:00:00 2001 From: Tobias McNulty Date: Tue, 5 Nov 2019 21:29:52 -0500 Subject: [PATCH] copy files from fixtures dir to MEDIA_ROOT (or remote storage) during load_initial_data (#255) Note: This change will require any existing deployments to manually copy bakerydemo/base/fixtures/media/original_images/ into bakerydemo/media/ --- .gitignore | 1 - .travis.yml | 3 +++ .../media/original_images/Akranes.jpg | Bin .../media/original_images/Anadama_bread_1.jpg | Bin ...served_with_Coconut_Milk_in_Tamil_Nadu.JPG | Bin .../media/original_images/Arepa_asada.JPG | Bin .../fixtures}/media/original_images/Bagel.jpg | Bin .../Baguette_de_pain_WikiCheese_Lausanne.jpg | Bin .../media/original_images/Bammies.jpg | Bin .../fixtures}/media/original_images/Bazin.jpg | Bin .../Bean-jam-bunanpankatori-cityjapan.JPG | Bin .../original_images/Bedouins_making_bread.jpg | Bin .../original_images/Belgische_waffeln.jpg | Bin .../media/original_images/Bhakri_1.jpg | Bin .../Bolani_Afghan_bread_01.jpg | Bin .../media/original_images/Breads_1.jpg | Bin .../Capture_decran_2017-03-23_a_15.16.46.png | Bin .../fixtures}/media/original_images/Hofn.jpg | Bin .../Olandshvedebrod_6082070226.jpg | Bin .../Sourdough_rye_with_walnuts.jpg | Bin .../aevar-gudmundsson-selfoss.jpg | Bin .../media/original_images/bakingsoda.jpg | Bin .../media/original_images/bostoncream.jpg | Bin .../media/original_images/bread5.jpg | Bin .../media/original_images/bread6.jpg | Bin .../media/original_images/breads1.jpg | Bin .../media/original_images/breads2.jpg | Bin .../media/original_images/breads3.jpg | Bin .../media/original_images/breads4.jpg | Bin .../hof-cornell-university-filter.jpg | Bin .../media/original_images/howlin.jpg | Bin .../original_images/lightnin_hopkins.jpg | Bin .../media/original_images/muddy_waters.jpg | Bin .../original_images/muddy_waters_hUPkmSW.jpg | Bin .../reykjavik-sverrir-thorolfsson.jpg | Bin .../media/original_images/robert_johnson2.jpg | Bin .../media/original_images/sliced.jpg | Bin .../media/original_images/soda_bread.jpg | Bin .../media/original_images/sprint_crew.jpg | Bin .../fixtures}/media/original_images/vik.jpg | Bin .../fixtures}/media/original_images/yeast.jpg | Bin .../management/commands/load_initial_data.py | 19 ++++++++++++++++++ bakerydemo/media/.gitignore | 4 ++++ docker-compose.yml | 4 ++++ readme.md | 6 ++---- requirements/production.txt | 3 --- 46 files changed, 32 insertions(+), 8 deletions(-) rename bakerydemo/{ => base/fixtures}/media/original_images/Akranes.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Anadama_bread_1.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Appam_served_with_Coconut_Milk_in_Tamil_Nadu.JPG (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Arepa_asada.JPG (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Bagel.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Baguette_de_pain_WikiCheese_Lausanne.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Bammies.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Bazin.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Bean-jam-bunanpankatori-cityjapan.JPG (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Bedouins_making_bread.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Belgische_waffeln.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Bhakri_1.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Bolani_Afghan_bread_01.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Breads_1.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Capture_decran_2017-03-23_a_15.16.46.png (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Hofn.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Olandshvedebrod_6082070226.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/Sourdough_rye_with_walnuts.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/aevar-gudmundsson-selfoss.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/bakingsoda.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/bostoncream.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/bread5.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/bread6.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/breads1.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/breads2.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/breads3.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/breads4.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/hof-cornell-university-filter.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/howlin.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/lightnin_hopkins.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/muddy_waters.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/muddy_waters_hUPkmSW.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/reykjavik-sverrir-thorolfsson.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/robert_johnson2.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/sliced.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/soda_bread.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/sprint_crew.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/vik.jpg (100%) rename bakerydemo/{ => base/fixtures}/media/original_images/yeast.jpg (100%) create mode 100644 bakerydemo/media/.gitignore diff --git a/.gitignore b/.gitignore index 2455fad..79662cf 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ /data .~* common/CACHE -bakerydemo/media/images/* bakerydemo/settings/local.py bakerydemodb __pycache__ diff --git a/.travis.yml b/.travis.yml index 679d240..5a21538 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,6 +22,9 @@ script: - docker-compose run app /bin/sh -c "/venv/bin/pip install flake8 && /venv/bin/flake8 ." - docker-compose run app /venv/bin/python /code/manage.py check - docker-compose run app /venv/bin/python /code/manage.py makemigrations --check + - docker-compose run app /venv/bin/python /code/manage.py load_initial_data + # Check for broken image links (wget will have a non-zero exit code if it encounters a 404) + - wget -m http://localhost:8000 after_script: - docker-compose logs diff --git a/bakerydemo/media/original_images/Akranes.jpg b/bakerydemo/base/fixtures/media/original_images/Akranes.jpg similarity index 100% rename from bakerydemo/media/original_images/Akranes.jpg rename to bakerydemo/base/fixtures/media/original_images/Akranes.jpg diff --git a/bakerydemo/media/original_images/Anadama_bread_1.jpg b/bakerydemo/base/fixtures/media/original_images/Anadama_bread_1.jpg similarity index 100% rename from bakerydemo/media/original_images/Anadama_bread_1.jpg rename to bakerydemo/base/fixtures/media/original_images/Anadama_bread_1.jpg diff --git a/bakerydemo/media/original_images/Appam_served_with_Coconut_Milk_in_Tamil_Nadu.JPG b/bakerydemo/base/fixtures/media/original_images/Appam_served_with_Coconut_Milk_in_Tamil_Nadu.JPG similarity index 100% rename from bakerydemo/media/original_images/Appam_served_with_Coconut_Milk_in_Tamil_Nadu.JPG rename to bakerydemo/base/fixtures/media/original_images/Appam_served_with_Coconut_Milk_in_Tamil_Nadu.JPG diff --git a/bakerydemo/media/original_images/Arepa_asada.JPG b/bakerydemo/base/fixtures/media/original_images/Arepa_asada.JPG similarity index 100% rename from bakerydemo/media/original_images/Arepa_asada.JPG rename to bakerydemo/base/fixtures/media/original_images/Arepa_asada.JPG diff --git a/bakerydemo/media/original_images/Bagel.jpg b/bakerydemo/base/fixtures/media/original_images/Bagel.jpg similarity index 100% rename from bakerydemo/media/original_images/Bagel.jpg rename to bakerydemo/base/fixtures/media/original_images/Bagel.jpg diff --git a/bakerydemo/media/original_images/Baguette_de_pain_WikiCheese_Lausanne.jpg b/bakerydemo/base/fixtures/media/original_images/Baguette_de_pain_WikiCheese_Lausanne.jpg similarity index 100% rename from bakerydemo/media/original_images/Baguette_de_pain_WikiCheese_Lausanne.jpg rename to bakerydemo/base/fixtures/media/original_images/Baguette_de_pain_WikiCheese_Lausanne.jpg diff --git a/bakerydemo/media/original_images/Bammies.jpg b/bakerydemo/base/fixtures/media/original_images/Bammies.jpg similarity index 100% rename from bakerydemo/media/original_images/Bammies.jpg rename to bakerydemo/base/fixtures/media/original_images/Bammies.jpg diff --git a/bakerydemo/media/original_images/Bazin.jpg b/bakerydemo/base/fixtures/media/original_images/Bazin.jpg similarity index 100% rename from bakerydemo/media/original_images/Bazin.jpg rename to bakerydemo/base/fixtures/media/original_images/Bazin.jpg diff --git a/bakerydemo/media/original_images/Bean-jam-bunanpankatori-cityjapan.JPG b/bakerydemo/base/fixtures/media/original_images/Bean-jam-bunanpankatori-cityjapan.JPG similarity index 100% rename from bakerydemo/media/original_images/Bean-jam-bunanpankatori-cityjapan.JPG rename to bakerydemo/base/fixtures/media/original_images/Bean-jam-bunanpankatori-cityjapan.JPG diff --git a/bakerydemo/media/original_images/Bedouins_making_bread.jpg b/bakerydemo/base/fixtures/media/original_images/Bedouins_making_bread.jpg similarity index 100% rename from bakerydemo/media/original_images/Bedouins_making_bread.jpg rename to bakerydemo/base/fixtures/media/original_images/Bedouins_making_bread.jpg diff --git a/bakerydemo/media/original_images/Belgische_waffeln.jpg b/bakerydemo/base/fixtures/media/original_images/Belgische_waffeln.jpg similarity index 100% rename from bakerydemo/media/original_images/Belgische_waffeln.jpg rename to bakerydemo/base/fixtures/media/original_images/Belgische_waffeln.jpg diff --git a/bakerydemo/media/original_images/Bhakri_1.jpg b/bakerydemo/base/fixtures/media/original_images/Bhakri_1.jpg similarity index 100% rename from bakerydemo/media/original_images/Bhakri_1.jpg rename to bakerydemo/base/fixtures/media/original_images/Bhakri_1.jpg diff --git a/bakerydemo/media/original_images/Bolani_Afghan_bread_01.jpg b/bakerydemo/base/fixtures/media/original_images/Bolani_Afghan_bread_01.jpg similarity index 100% rename from bakerydemo/media/original_images/Bolani_Afghan_bread_01.jpg rename to bakerydemo/base/fixtures/media/original_images/Bolani_Afghan_bread_01.jpg diff --git a/bakerydemo/media/original_images/Breads_1.jpg b/bakerydemo/base/fixtures/media/original_images/Breads_1.jpg similarity index 100% rename from bakerydemo/media/original_images/Breads_1.jpg rename to bakerydemo/base/fixtures/media/original_images/Breads_1.jpg diff --git a/bakerydemo/media/original_images/Capture_decran_2017-03-23_a_15.16.46.png b/bakerydemo/base/fixtures/media/original_images/Capture_decran_2017-03-23_a_15.16.46.png similarity index 100% rename from bakerydemo/media/original_images/Capture_decran_2017-03-23_a_15.16.46.png rename to bakerydemo/base/fixtures/media/original_images/Capture_decran_2017-03-23_a_15.16.46.png diff --git a/bakerydemo/media/original_images/Hofn.jpg b/bakerydemo/base/fixtures/media/original_images/Hofn.jpg similarity index 100% rename from bakerydemo/media/original_images/Hofn.jpg rename to bakerydemo/base/fixtures/media/original_images/Hofn.jpg diff --git a/bakerydemo/media/original_images/Olandshvedebrod_6082070226.jpg b/bakerydemo/base/fixtures/media/original_images/Olandshvedebrod_6082070226.jpg similarity index 100% rename from bakerydemo/media/original_images/Olandshvedebrod_6082070226.jpg rename to bakerydemo/base/fixtures/media/original_images/Olandshvedebrod_6082070226.jpg diff --git a/bakerydemo/media/original_images/Sourdough_rye_with_walnuts.jpg b/bakerydemo/base/fixtures/media/original_images/Sourdough_rye_with_walnuts.jpg similarity index 100% rename from bakerydemo/media/original_images/Sourdough_rye_with_walnuts.jpg rename to bakerydemo/base/fixtures/media/original_images/Sourdough_rye_with_walnuts.jpg diff --git a/bakerydemo/media/original_images/aevar-gudmundsson-selfoss.jpg b/bakerydemo/base/fixtures/media/original_images/aevar-gudmundsson-selfoss.jpg similarity index 100% rename from bakerydemo/media/original_images/aevar-gudmundsson-selfoss.jpg rename to bakerydemo/base/fixtures/media/original_images/aevar-gudmundsson-selfoss.jpg diff --git a/bakerydemo/media/original_images/bakingsoda.jpg b/bakerydemo/base/fixtures/media/original_images/bakingsoda.jpg similarity index 100% rename from bakerydemo/media/original_images/bakingsoda.jpg rename to bakerydemo/base/fixtures/media/original_images/bakingsoda.jpg diff --git a/bakerydemo/media/original_images/bostoncream.jpg b/bakerydemo/base/fixtures/media/original_images/bostoncream.jpg similarity index 100% rename from bakerydemo/media/original_images/bostoncream.jpg rename to bakerydemo/base/fixtures/media/original_images/bostoncream.jpg diff --git a/bakerydemo/media/original_images/bread5.jpg b/bakerydemo/base/fixtures/media/original_images/bread5.jpg similarity index 100% rename from bakerydemo/media/original_images/bread5.jpg rename to bakerydemo/base/fixtures/media/original_images/bread5.jpg diff --git a/bakerydemo/media/original_images/bread6.jpg b/bakerydemo/base/fixtures/media/original_images/bread6.jpg similarity index 100% rename from bakerydemo/media/original_images/bread6.jpg rename to bakerydemo/base/fixtures/media/original_images/bread6.jpg diff --git a/bakerydemo/media/original_images/breads1.jpg b/bakerydemo/base/fixtures/media/original_images/breads1.jpg similarity index 100% rename from bakerydemo/media/original_images/breads1.jpg rename to bakerydemo/base/fixtures/media/original_images/breads1.jpg diff --git a/bakerydemo/media/original_images/breads2.jpg b/bakerydemo/base/fixtures/media/original_images/breads2.jpg similarity index 100% rename from bakerydemo/media/original_images/breads2.jpg rename to bakerydemo/base/fixtures/media/original_images/breads2.jpg diff --git a/bakerydemo/media/original_images/breads3.jpg b/bakerydemo/base/fixtures/media/original_images/breads3.jpg similarity index 100% rename from bakerydemo/media/original_images/breads3.jpg rename to bakerydemo/base/fixtures/media/original_images/breads3.jpg diff --git a/bakerydemo/media/original_images/breads4.jpg b/bakerydemo/base/fixtures/media/original_images/breads4.jpg similarity index 100% rename from bakerydemo/media/original_images/breads4.jpg rename to bakerydemo/base/fixtures/media/original_images/breads4.jpg diff --git a/bakerydemo/media/original_images/hof-cornell-university-filter.jpg b/bakerydemo/base/fixtures/media/original_images/hof-cornell-university-filter.jpg similarity index 100% rename from bakerydemo/media/original_images/hof-cornell-university-filter.jpg rename to bakerydemo/base/fixtures/media/original_images/hof-cornell-university-filter.jpg diff --git a/bakerydemo/media/original_images/howlin.jpg b/bakerydemo/base/fixtures/media/original_images/howlin.jpg similarity index 100% rename from bakerydemo/media/original_images/howlin.jpg rename to bakerydemo/base/fixtures/media/original_images/howlin.jpg diff --git a/bakerydemo/media/original_images/lightnin_hopkins.jpg b/bakerydemo/base/fixtures/media/original_images/lightnin_hopkins.jpg similarity index 100% rename from bakerydemo/media/original_images/lightnin_hopkins.jpg rename to bakerydemo/base/fixtures/media/original_images/lightnin_hopkins.jpg diff --git a/bakerydemo/media/original_images/muddy_waters.jpg b/bakerydemo/base/fixtures/media/original_images/muddy_waters.jpg similarity index 100% rename from bakerydemo/media/original_images/muddy_waters.jpg rename to bakerydemo/base/fixtures/media/original_images/muddy_waters.jpg diff --git a/bakerydemo/media/original_images/muddy_waters_hUPkmSW.jpg b/bakerydemo/base/fixtures/media/original_images/muddy_waters_hUPkmSW.jpg similarity index 100% rename from bakerydemo/media/original_images/muddy_waters_hUPkmSW.jpg rename to bakerydemo/base/fixtures/media/original_images/muddy_waters_hUPkmSW.jpg diff --git a/bakerydemo/media/original_images/reykjavik-sverrir-thorolfsson.jpg b/bakerydemo/base/fixtures/media/original_images/reykjavik-sverrir-thorolfsson.jpg similarity index 100% rename from bakerydemo/media/original_images/reykjavik-sverrir-thorolfsson.jpg rename to bakerydemo/base/fixtures/media/original_images/reykjavik-sverrir-thorolfsson.jpg diff --git a/bakerydemo/media/original_images/robert_johnson2.jpg b/bakerydemo/base/fixtures/media/original_images/robert_johnson2.jpg similarity index 100% rename from bakerydemo/media/original_images/robert_johnson2.jpg rename to bakerydemo/base/fixtures/media/original_images/robert_johnson2.jpg diff --git a/bakerydemo/media/original_images/sliced.jpg b/bakerydemo/base/fixtures/media/original_images/sliced.jpg similarity index 100% rename from bakerydemo/media/original_images/sliced.jpg rename to bakerydemo/base/fixtures/media/original_images/sliced.jpg diff --git a/bakerydemo/media/original_images/soda_bread.jpg b/bakerydemo/base/fixtures/media/original_images/soda_bread.jpg similarity index 100% rename from bakerydemo/media/original_images/soda_bread.jpg rename to bakerydemo/base/fixtures/media/original_images/soda_bread.jpg diff --git a/bakerydemo/media/original_images/sprint_crew.jpg b/bakerydemo/base/fixtures/media/original_images/sprint_crew.jpg similarity index 100% rename from bakerydemo/media/original_images/sprint_crew.jpg rename to bakerydemo/base/fixtures/media/original_images/sprint_crew.jpg diff --git a/bakerydemo/media/original_images/vik.jpg b/bakerydemo/base/fixtures/media/original_images/vik.jpg similarity index 100% rename from bakerydemo/media/original_images/vik.jpg rename to bakerydemo/base/fixtures/media/original_images/vik.jpg diff --git a/bakerydemo/media/original_images/yeast.jpg b/bakerydemo/base/fixtures/media/original_images/yeast.jpg similarity index 100% rename from bakerydemo/media/original_images/yeast.jpg rename to bakerydemo/base/fixtures/media/original_images/yeast.jpg diff --git a/bakerydemo/base/management/commands/load_initial_data.py b/bakerydemo/base/management/commands/load_initial_data.py index 711f171..9896641 100644 --- a/bakerydemo/base/management/commands/load_initial_data.py +++ b/bakerydemo/base/management/commands/load_initial_data.py @@ -1,6 +1,7 @@ import os from django.conf import settings +from django.core.files.storage import default_storage, FileSystemStorage from django.core.management.base import BaseCommand from django.core.management import call_command @@ -8,10 +9,28 @@ from wagtail.core.models import Site, Page class Command(BaseCommand): + def _copy_files(self, local_storage, path): + """ + Recursively copy files from local_storage to default_storage. Used + to automatically bootstrap the media directory (both locally and on + cloud providers) with the images linked from the initial data (and + included in MEDIA_ROOT). + """ + directories, file_names = local_storage.listdir(path) + for directory in directories: + self._copy_files(local_storage, path + directory + '/') + for file_name in file_names: + with local_storage.open(path + file_name) as file_: + default_storage.save(path + file_name, file_) + def handle(self, **options): fixtures_dir = os.path.join(settings.PROJECT_DIR, 'base', 'fixtures') fixture_file = os.path.join(fixtures_dir, 'bakerydemo.json') + print("Copying media files to configured storage...") + local_storage = FileSystemStorage(os.path.join(fixtures_dir, 'media')) + self._copy_files(local_storage, '') # file storage paths are relative + # Wagtail creates default Site and Page instances during install, but we already have # them in the data load. Remove the auto-generated ones. if Site.objects.filter(hostname='localhost').exists(): diff --git a/bakerydemo/media/.gitignore b/bakerydemo/media/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/bakerydemo/media/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/docker-compose.yml b/docker-compose.yml index b96f0b4..ad0efae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,8 @@ services: build: context: . dockerfile: ./Dockerfile + volumes: + - media-root:/code/bakerydemo/media links: - db:db - redis:redis @@ -39,3 +41,5 @@ services: - db - redis - elasticsearch +volumes: + media-root: diff --git a/readme.md b/readme.md index 18cf38c..95b240a 100644 --- a/readme.md +++ b/readme.md @@ -188,10 +188,8 @@ environment, set the same environment variables there using the method appropria Once Heroku restarts your application or your Docker container is refreshed, you should have persistent media storage! -To copy the initial data included with this demo to the S3 bucket (assuming you ran `./manage.py load_initial_data` per -the above), you can use the AWS CLI included with the requirements: - - heroku run aws s3 sync bakerydemo/media/original_images/ s3:///original_images/ +Running `./manage.py load_initial_data` will copy local images to S3, but if you set up S3 after you ran it the first +time you might need to run it again. # Next steps diff --git a/requirements/production.txt b/requirements/production.txt index fa73b79..9603704 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -15,6 +15,3 @@ botocore==1.12.33 aws-requests-auth==0.4.0 django-redis==4.8.0 django_cache_url==2.0.0 -# For copying initial media to S3 bucket -awscli==1.16.43 -urllib3>=1.21.1,<1.26