diff --git a/.flake8 b/.flake8 deleted file mode 100644 index c295ba9..0000000 --- a/.flake8 +++ /dev/null @@ -1,7 +0,0 @@ -[flake8] - -max-line-length = 120 -# W503: 'and' on start of line -ignore = W503 - -; vim:set ft=dosini: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e45fdaa --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,79 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + +jobs: + test: + name: "Python ${{ matrix.python-version }} / Django ${{ matrix.django-version }}" + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + include: + - python-version: "3.11" + django-version: "django5" + toxenv: "py311-django5" + - python-version: "3.12" + django-version: "django5" + toxenv: "py312-django5" + - python-version: "3.13" + django-version: "django5" + toxenv: "py313-django5" + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e ${{ matrix.toxenv }} + + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: "pip" + + - name: Install tox + run: pip install tox + + - name: Run linting + run: tox -e lint + + prodsettings: + name: Check production settings + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: "pip" + + - name: Install tox + run: pip install tox + + - name: Check deploy settings + run: tox -e prodsettings diff --git a/.gitignore b/.gitignore index 8d8d9f5..fd8f7a9 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ # Dev-related files dev/ +/.ruff_cache/ /static/ /.tox/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9b05ebb..0000000 --- a/.travis.yml +++ /dev/null @@ -1,50 +0,0 @@ -sudo: false -language: python -cache: pip -dist: bionic - -branches: - only: - - master - -script: - - tox - -install: - - pip install tox - -matrix: - include: - - python: "3.6" - env: TOXENV=py36-django111 - - python: "3.6" - env: TOXENV=py36-django20 - - python: "3.7" - env: TOXENV=py37-django20 - - python: "3.6" - env: TOXENV=py36-django21 - - python: "3.7" - env: TOXENV=py37-django21 - - python: "3.6" - env: TOXENV=py36-django22 - - python: "3.7" - env: TOXENV=py37-django22 - - # Linting - - python: "3.7" - env: TOXENV=lint - # Test dev makefile - - python: "3.7" - env: TOXENV=dev - # Check default production settings - - python: "3.7" - env: TOXENV=prodsettings - -notifications: - email: false - irc: - channels: - - "irc.freenode.org#platal" - on_success: change - on_failure: always - use_notice: true diff --git a/MANIFEST.in b/MANIFEST.in index 8c2146e..8c99b4b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include LICENSE README.rst -include requirements_dev.txt release.txt +include requirements_dev.txt release.txt pyproject.toml graft xorgdata @@ -10,4 +10,4 @@ recursive-include third_party *.po global-exclude *.py[cod] __pycache__ *.so .*.swp -exclude Makefile example_* manage.py tox.ini .flake8 .travis.yml +exclude Makefile example_* manage.py tox.ini diff --git a/Makefile b/Makefile index 2e39767..4081af0 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,19 @@ -PACKAGE=xorgdata -SRC_DIR=$(PACKAGE) -TESTS_DIR=tests -DOC_DIR=docs - -# Use current python binary instead of system default. -COVERAGE = python $(shell which coverage) -FLAKE8 = flake8 -DJANGO_ADMIN = django-admin.py +PACKAGE = xorgdata +SRC_DIR = $(PACKAGE) +TESTS_DIR = tests + +# Utilise le binaire Python courant +COVERAGE = python -m coverage +RUFF = ruff +DJANGO_ADMIN = django-admin + PO_FILES = $(shell find $(SRC_DIR) -name '*.po') MO_FILES = $(PO_FILES:.po=.mo) all: default - default: build - clean: find . -type f -name '*.pyc' -delete find $(SRC_DIR) $(TESTS_DIR) -type f -path '*/__pycache__/*' -delete @@ -49,20 +47,17 @@ test: build checkdeploy: python manage.py check --deploy --fail-level WARNING - lint: check-manifest - $(FLAKE8) --config .flake8 $(SRC_DIR) - $(FLAKE8) --config .flake8 $(TESTS_DIR) + $(RUFF) check $(SRC_DIR) $(TESTS_DIR) + +format: + $(RUFF) format $(SRC_DIR) $(TESTS_DIR) coverage: $(COVERAGE) erase - $(COVERAGE) run "--include=$(SRC_DIR)/*.py,$(TESTS_DIR)/*.py" --branch manage.py test $(TESTS_DIR) - $(COVERAGE) report "--include=$(SRC_DIR)/*.py,$(TESTS_DIR)/*.py" - $(COVERAGE) html "--include=$(SRC_DIR)/*.py,$(TESTS_DIR)/*.py" - -doc: - $(MAKE) -C $(DOC_DIR) html - + $(COVERAGE) run --branch manage.py test $(TESTS_DIR) + $(COVERAGE) report + $(COVERAGE) html -.PHONY: all checkdeploy clean coverage createdb default doc install-deps lint poupdate test testall update +.PHONY: all checkdeploy clean coverage createdb default doc format lint poupdate test testall update diff --git a/README.rst b/README.rst index ef7093e..9659f36 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,9 @@ xorgdata ======== -.. image:: https://secure.travis-ci.org/Polytechnique-org/xorgdata.png?branch=master - :target: http://travis-ci.org/Polytechnique-org/xorgdata/ +.. image:: https://github.com/Polytechnique-org/xorgdata/actions/workflows/ci.yml/badge.svg?branch=master + :target: https://github.com/Polytechnique-org/xorgdata/actions/workflows/ci.yml + :alt: CI xorgdata handles the central data store for Polytechnique.org services. It pulls data from AX's website and pushes it to Polytechnique.org's services. @@ -19,7 +20,7 @@ Here are some commands to set up a development environment: .. code-block:: sh # Create a new virtual env - pew new xorgdata + python -m venv xorgdata # Install dependencies make update diff --git a/manage.py b/manage.py index 5a0106f..415ae82 100755 --- a/manage.py +++ b/manage.py @@ -1,8 +1,12 @@ #!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" + import os import sys -if __name__ == "__main__": + +def main(): + """Run administrative tasks.""" os.environ.setdefault("DJANGO_SETTINGS_MODULE", "xorgdata.settings") try: from django.core.management import execute_from_command_line @@ -13,3 +17,7 @@ "forget to activate a virtual environment?" ) from exc execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7b7a9fa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "xorgdata" +dynamic = ["version"] +description = "Polytechnique.org central datastore" +readme = "README.rst" +license = { text = "AGPL-3.0" } +authors = [ + { name = "Polytechnique.org dev team", email = "devel+xorgdata@staff.polytechnique.org" }, +] +requires-python = ">=3.11" +dependencies = [ + "Django~=5.2", + "getconf", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Web Environment", + "Framework :: Django", + "License :: OSI Approved :: GNU Affero General Public License v3", + "Natural Language :: French", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Internet :: WWW/HTTP :: Session", + "Topic :: System :: Systems Administration :: Authentication/Directory", +] + +[project.urls] +Homepage = "https://github.com/Polytechnique-org/xorgdata" + +[tool.setuptools.dynamic] +version = { attr = "xorgdata.__version__" } + +[tool.setuptools.packages.find] +include = ["xorgdata", "xorgdata.*"] + +# --------------------------------------------------------------------------- +# Ruff +# --------------------------------------------------------------------------- +[tool.ruff] +line-length = 119 +target-version = "py311" +exclude = ["migrations"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I"] +ignore = [] + +# --------------------------------------------------------------------------- +# Coverage +# --------------------------------------------------------------------------- +[tool.coverage.run] +source = ["xorgdata"] +branch = true + +[tool.coverage.report] +show_missing = true diff --git a/requirements_dev.txt b/requirements_dev.txt index db9d077..6ac1632 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,13 @@ -e . -check_manifest -flake8 -psycopg2 +# Linting (remplace flake8 par ruff, plus rapide) +ruff>=0.4.0 + +# Vérification du MANIFEST +check-manifest>=0.49 + +# Base de données PostgreSQL +psycopg2>=2.9 + +# Tests & couverture +coverage[toml]>=7.4 diff --git a/setup.py b/setup.py deleted file mode 100644 index bd8c2d5..0000000 --- a/setup.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# This code is distributed under the GPLv3 license. -# Copyright (c) Polytechnique.org - - -import codecs -import os -import re - -from setuptools import find_packages, setup - -root_dir = os.path.abspath(os.path.dirname(__file__)) - - -def get_version(package_name): - version_re = re.compile(r"^__version__ = [\"']([\w_.-]+)[\"']$") - package_components = package_name.split('.') - init_path = os.path.join(root_dir, *(package_components + ['__init__.py'])) - with codecs.open(init_path, 'r', 'utf-8') as f: - for line in f: - match = version_re.match(line[:-1]) - if match: - return match.groups()[0] - return '0.1.1' - -PACKAGE = 'xorgdata' - -setup( - name=PACKAGE, - version=get_version(PACKAGE), - author="Polytechnique.org dev team", - author_email="devel+xorgdata@staff.polytechnique.org", - description="Polytechnique.org central datastore", - license='AGPLv3', - url='https://github.com/Polytechnique-org/%s' % PACKAGE, - - packages=find_packages(include=[PACKAGE, '%s.*' % PACKAGE]), - include_package_data=True, - - python_requires='>=3.4.2', - install_requires=[ - 'Django>=2.0,<2.3', - 'getconf', - ], - setup_requires=[ - 'setuptools>=0.8', - ], - - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Environment :: Web Environment', - 'Framework :: Django', - 'License :: OSI Approved :: GNU Affero General Public License v3', - 'Natural Language :: French', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3 :: Only', - 'Topic :: Internet :: WWW/HTTP :: Session', - 'Topic :: System :: Systems Administration :: Authentication/Directory' - ], -) diff --git a/tests/test_exportforauth.py b/tests/test_exportforauth.py index 15385d3..8143427 100644 --- a/tests/test_exportforauth.py +++ b/tests/test_exportforauth.py @@ -1,50 +1,53 @@ import json +from io import StringIO from django.core.management import call_command from django.test import TestCase -from django.utils.six import StringIO -from .test_importcsv import TEST_CSV_PATHS from xorgdata.alumnforce.models import Account +from .test_importcsv import TEST_CSV_PATHS + class ExportForAuthTests(TestCase): - def test_exportforauth(self): # Start by populating the database for file_path in TEST_CSV_PATHS.values(): - call_command('importcsv', file_path, verbosity=0) + call_command("importcsv", file_path, verbosity=0) # Export the database out = StringIO() - call_command('exportforauth', stdout=out) + call_command("exportforauth", stdout=out) exported_data = json.loads(out.getvalue()) - self.assertEqual(exported_data, [ - { - 'xorg_id': 'louis.vaneau.1829', - 'af_id': 1, - 'ax_contributor': False, - 'axjr_subscribed': False, - 'last_updated': '2001-02-03', - } - ]) + self.assertEqual( + exported_data, + [ + { + "xorg_id": "louis.vaneau.1829", + "af_id": 1, + "ax_contributor": False, + "axjr_subscribed": False, + "last_updated": "2001-02-03", + } + ], + ) # Change the role of the user and verify that it gets propagated in the export account = Account.objects.get(af_id=1) - account.additional_roles = '2,5' # graduated, contributor + account.additional_roles = "2,5" # graduated, contributor account.save() out = StringIO() - call_command('exportforauth', stdout=out) + call_command("exportforauth", stdout=out) exported_data = json.loads(out.getvalue()) - self.assertEqual(exported_data[0]['af_id'], 1) - self.assertEqual(exported_data[0]['ax_contributor'], True) - self.assertEqual(exported_data[0]['axjr_subscribed'], False) + self.assertEqual(exported_data[0]["af_id"], 1) + self.assertEqual(exported_data[0]["ax_contributor"], True) + self.assertEqual(exported_data[0]["axjr_subscribed"], False) - account.additional_roles = '5,7,17' # contributor, student, subscribed + account.additional_roles = "5,7,17" # contributor, student, subscribed account.save() out = StringIO() - call_command('exportforauth', stdout=out) + call_command("exportforauth", stdout=out) exported_data = json.loads(out.getvalue()) - self.assertEqual(exported_data[0]['af_id'], 1) - self.assertEqual(exported_data[0]['ax_contributor'], True) - self.assertEqual(exported_data[0]['axjr_subscribed'], True) + self.assertEqual(exported_data[0]["af_id"], 1) + self.assertEqual(exported_data[0]["ax_contributor"], True) + self.assertEqual(exported_data[0]["axjr_subscribed"], True) diff --git a/tests/test_importallusers.py b/tests/test_importallusers.py index e55bb7e..7070ae7 100644 --- a/tests/test_importallusers.py +++ b/tests/test_importallusers.py @@ -1,26 +1,27 @@ import datetime -from pathlib import Path import re +from io import StringIO +from pathlib import Path from django.core.management import call_command from django.test import TestCase -from django.utils.six import StringIO from xorgdata.alumnforce.models import Account class ImportAllUsersTests(TestCase): """Test importing a full dump of the users""" + def setUp(self): - csv_dirpath = Path(__file__).parent / 'files' - self.csv_file = csv_dirpath / 'export-users-20010203-040506.csv' + csv_dirpath = Path(__file__).parent / "files" + self.csv_file = csv_dirpath / "export-users-20010203-040506.csv" @staticmethod def count_error_lines_from_stdout(out): num_error_lines = 0 for out_line in out.getvalue().splitlines(): # Remove color escape sequences - line = out_line.replace('\x1b[32;1m', '').replace('\x1b[0m', '') + line = out_line.replace("\x1b[32;1m", "").replace("\x1b[0m", "") if re.match(r"^Loaded 2 values from full export '", line): continue else: # pragma: no cover @@ -31,7 +32,7 @@ def count_error_lines_from_stdout(out): def test_import_all_users(self): out = StringIO() - call_command('importallusers', self.csv_file, stdout=out) + call_command("importallusers", self.csv_file, stdout=out) self.assertEqual(self.count_error_lines_from_stdout(out), 0) # Ensure the loaded data is correct @@ -39,107 +40,104 @@ def test_import_all_users(self): self.assertIsNotNone(user) self.assertEqual(user.af_id, 1) self.assertEqual(user.ax_id, None) - self.assertEqual(user.first_name, 'admin') - self.assertEqual(user.last_name, '') - self.assertEqual(user.common_name, 'alumnforce') - self.assertEqual(user.civility, '') + self.assertEqual(user.first_name, "admin") + self.assertEqual(user.last_name, "") + self.assertEqual(user.common_name, "alumnforce") + self.assertEqual(user.civility, "") self.assertEqual(user.birthdate, None) - self.assertEqual(user.address_1, 'appt du Test') - self.assertEqual(user.address_2, 'rue du Test') - self.assertEqual(user.address_3, '') - self.assertEqual(user.address_4, '') - self.assertEqual(user.address_postcode, '75000') - self.assertEqual(user.address_city, 'Paris') - self.assertEqual(user.address_state, '') - self.assertEqual(user.address_country, 'FR') + self.assertEqual(user.address_1, "appt du Test") + self.assertEqual(user.address_2, "rue du Test") + self.assertEqual(user.address_3, "") + self.assertEqual(user.address_4, "") + self.assertEqual(user.address_postcode, "75000") + self.assertEqual(user.address_city, "Paris") + self.assertEqual(user.address_state, "") + self.assertEqual(user.address_country, "FR") self.assertEqual(user.address_npai, False) - self.assertEqual(user.phone_personnal, '') - self.assertEqual(user.phone_mobile, '') - self.assertEqual(user.email_1, 'noreply@alumnforce.com') - self.assertEqual(user.email_2, '') - self.assertEqual(user.nationality, '') - self.assertEqual(user.nationality_2, '') - self.assertEqual(user.nationality_3, '') + self.assertEqual(user.phone_personnal, "") + self.assertEqual(user.phone_mobile, "") + self.assertEqual(user.email_1, "noreply@alumnforce.com") + self.assertEqual(user.email_2, "") + self.assertEqual(user.nationality, "") + self.assertEqual(user.nationality_2, "") + self.assertEqual(user.nationality_3, "") self.assertEqual(user.dead, False) self.assertEqual(user.deathdate, None) - self.assertEqual(user.dead_for_france, '') + self.assertEqual(user.dead_for_france, "") self.assertEqual(user.user_kind, 3) - self.assertEqual(user.additional_roles, '2,3,5') - self.assertEqual(user.xorg_id, 'admin.alumnforce') - self.assertEqual(user.school_id, '') - self.assertEqual(user.admission_path, '') - self.assertEqual(user.cursus_domain, '') - self.assertEqual(user.cursus_name, '') - self.assertEqual(user.corps_current, '') - self.assertEqual(user.corps_origin, '') - self.assertEqual(user.corps_grade, '') - self.assertEqual(user.nickname, 'AlumnForce') - self.assertEqual(user.sport_section, '') - self.assertEqual(user.binets, '') - self.assertEqual(user.mail_reception, '') - self.assertEqual(user.newsletter_inscriptions, '') - self.assertEqual(user.profile_picture_url, '') + self.assertEqual(user.additional_roles, "2,3,5") + self.assertEqual(user.xorg_id, "admin.alumnforce") + self.assertEqual(user.school_id, "") + self.assertEqual(user.admission_path, "") + self.assertEqual(user.cursus_domain, "") + self.assertEqual(user.cursus_name, "") + self.assertEqual(user.corps_current, "") + self.assertEqual(user.corps_origin, "") + self.assertEqual(user.corps_grade, "") + self.assertEqual(user.nickname, "AlumnForce") + self.assertEqual(user.sport_section, "") + self.assertEqual(user.binets, "") + self.assertEqual(user.mail_reception, "") + self.assertEqual(user.newsletter_inscriptions, "") + self.assertEqual(user.profile_picture_url, "") self.assertEqual(user.last_update, datetime.date(2001, 2, 3)) self.assertEqual(user.deleted_since, None) user = Account.objects.get(af_id=2) self.assertIsNotNone(user) self.assertEqual(user.af_id, 2) - self.assertEqual(user.ax_id, '18290001') - self.assertEqual(user.first_name, 'Louis') - self.assertEqual(user.last_name, 'Vaneau') - self.assertEqual(user.common_name, 'Vaneau') - self.assertEqual(user.civility, 'M') + self.assertEqual(user.ax_id, "18290001") + self.assertEqual(user.first_name, "Louis") + self.assertEqual(user.last_name, "Vaneau") + self.assertEqual(user.common_name, "Vaneau") + self.assertEqual(user.civility, "M") self.assertEqual(user.birthdate, datetime.date(1811, 3, 27)) - self.assertEqual(user.address_1, 'rue de Babylone') - self.assertEqual(user.address_2, '') - self.assertEqual(user.address_3, '') - self.assertEqual(user.address_4, '') - self.assertEqual(user.address_postcode, '75007') - self.assertEqual(user.address_city, 'PARIS') - self.assertEqual(user.address_state, '') - self.assertEqual(user.address_country, 'FR') + self.assertEqual(user.address_1, "rue de Babylone") + self.assertEqual(user.address_2, "") + self.assertEqual(user.address_3, "") + self.assertEqual(user.address_4, "") + self.assertEqual(user.address_postcode, "75007") + self.assertEqual(user.address_city, "PARIS") + self.assertEqual(user.address_state, "") + self.assertEqual(user.address_country, "FR") self.assertEqual(user.address_npai, False) - self.assertEqual(user.phone_personnal, '06 00 00 00 00') - self.assertEqual(user.phone_mobile, '06 00 00 00 00') - self.assertEqual(user.email_1, 'louis.vaneau.1829@polytechnique.org') - self.assertEqual(user.email_2, 'louis.vaneau.1829+ax@polytechnique.org') - self.assertEqual(user.nationality, 'France') - self.assertEqual(user.nationality_2, '') - self.assertEqual(user.nationality_3, '') + self.assertEqual(user.phone_personnal, "06 00 00 00 00") + self.assertEqual(user.phone_mobile, "06 00 00 00 00") + self.assertEqual(user.email_1, "louis.vaneau.1829@polytechnique.org") + self.assertEqual(user.email_2, "louis.vaneau.1829+ax@polytechnique.org") + self.assertEqual(user.nationality, "France") + self.assertEqual(user.nationality_2, "") + self.assertEqual(user.nationality_3, "") self.assertEqual(user.dead, True) self.assertEqual(user.deathdate, datetime.date(1830, 7, 29)) - self.assertEqual(user.dead_for_france, '') + self.assertEqual(user.dead_for_france, "") self.assertEqual(user.user_kind, 1) - self.assertEqual(user.additional_roles, '') - self.assertEqual(user.xorg_id, 'louis.vaneau.1829') - self.assertEqual(user.school_id, '18290001') - self.assertEqual(user.admission_path, '') - self.assertEqual(user.cursus_domain, '') - self.assertEqual(user.cursus_name, '') - self.assertEqual(user.corps_current, 'Aucun (anc. démissionnaire)') - self.assertEqual(user.corps_origin, '') - self.assertEqual(user.corps_grade, 'Aucun') - self.assertEqual(user.nickname, '') - self.assertEqual(user.sport_section, 'Escrime') - self.assertEqual(user.binets, 'Binet Escrime') - self.assertEqual(user.mail_reception, '') - self.assertEqual(user.newsletter_inscriptions, 'Lettre mensuelle de Polytechnique.org,Lettre de la communauté') - self.assertEqual(user.profile_picture_url, '') + self.assertEqual(user.additional_roles, "") + self.assertEqual(user.xorg_id, "louis.vaneau.1829") + self.assertEqual(user.school_id, "18290001") + self.assertEqual(user.admission_path, "") + self.assertEqual(user.cursus_domain, "") + self.assertEqual(user.cursus_name, "") + self.assertEqual(user.corps_current, "Aucun (anc. démissionnaire)") + self.assertEqual(user.corps_origin, "") + self.assertEqual(user.corps_grade, "Aucun") + self.assertEqual(user.nickname, "") + self.assertEqual(user.sport_section, "Escrime") + self.assertEqual(user.binets, "Binet Escrime") + self.assertEqual(user.mail_reception, "") + self.assertEqual(user.newsletter_inscriptions, "Lettre mensuelle de Polytechnique.org,Lettre de la communauté") + self.assertEqual(user.profile_picture_url, "") self.assertEqual(user.last_update, datetime.date(2001, 2, 3)) self.assertEqual(user.deleted_since, None) def test_delete_user(self): # Create a user which will be destroyed - account = Account.objects.create( - af_id=123456, - user_kind=1, - last_update=datetime.date(2001, 2, 3)) + account = Account.objects.create(af_id=123456, user_kind=1, last_update=datetime.date(2001, 2, 3)) self.assertEqual(account.deleted_since, None) # Load exported data out = StringIO() - call_command('importallusers', self.csv_file, stdout=out) + call_command("importallusers", self.csv_file, stdout=out) self.assertEqual(self.count_error_lines_from_stdout(out), 0) # Check whether the account has been marked as deleted diff --git a/tests/test_importcsv.py b/tests/test_importcsv.py index b08f9da..6d545f5 100644 --- a/tests/test_importcsv.py +++ b/tests/test_importcsv.py @@ -1,39 +1,38 @@ import collections import datetime -from pathlib import Path import re +from io import StringIO +from pathlib import Path from django.core.management import call_command from django.test import TestCase -from django.utils.six import StringIO from xorgdata.alumnforce.models import Account, Group, ImportLog - TEST_CSV_FILES = ( - ('users', 'exportusers-afbo-Polytechnique-X-20010203.csv'), - ('userdegrees', 'exportuserdegrees-afbo-Polytechnique-X-20010203.csv'), - ('userjobs', 'exportuserjobs-afbo-Polytechnique-X-20010203.csv'), - ('groups', 'exportgroups-afbo-Polytechnique-X-20010203.csv'), - ('groupmembers', 'exportgroupmembers-afbo-Polytechnique-X-20010203.csv'), + ("users", "exportusers-afbo-Polytechnique-X-20010203.csv"), + ("userdegrees", "exportuserdegrees-afbo-Polytechnique-X-20010203.csv"), + ("userjobs", "exportuserjobs-afbo-Polytechnique-X-20010203.csv"), + ("groups", "exportgroups-afbo-Polytechnique-X-20010203.csv"), + ("groupmembers", "exportgroupmembers-afbo-Polytechnique-X-20010203.csv"), ) TEST_CSV_PATHS = collections.OrderedDict( - (kind, Path(__file__).parent / 'files' / file_name) - for kind, file_name in TEST_CSV_FILES + (kind, Path(__file__).parent / "files" / file_name) for kind, file_name in TEST_CSV_FILES ) class ImportCsvTests(TestCase): """Test importing several CSV files""" + def test_importcsv_verbose(self): for kind, file_path in TEST_CSV_PATHS.items(): out = StringIO() - call_command('importcsv', file_path, stdout=out) + call_command("importcsv", file_path, stdout=out) num_error_lines = 0 for out_line in out.getvalue().splitlines(): # Remove color escape sequences - line = out_line.replace('\x1b[32;1m', '').replace('\x1b[0m', '') - if re.match(r"^Loaded [0-9]+ values from " + kind + " '.*'$", line): + line = out_line.replace("\x1b[32;1m", "").replace("\x1b[0m", "") + if re.match(r"^Loaded [0-9]+ values from " + kind + r" '.*'\.$", line): continue else: # pragma: no cover # Display the errors @@ -48,19 +47,19 @@ def test_importcsv_verbose(self): self.assertEqual(import_log.is_incremental, True) self.assertEqual(import_log.error, ImportLog.SUCCESS) self.assertGreater(import_log.num_modified, 0) - self.assertNotEqual(import_log.message, '') + self.assertNotEqual(import_log.message, "") def test_importcsv_quiet(self): for file_path in TEST_CSV_PATHS.values(): out = StringIO() - call_command('importcsv', file_path, stdout=out, verbosity=0) - self.assertEqual(out.getvalue(), '') + call_command("importcsv", file_path, stdout=out, verbosity=0) + self.assertEqual(out.getvalue(), "") def test_importcsv_with_kind(self): for kind, file_path in TEST_CSV_PATHS.items(): out = StringIO() - call_command('importcsv', '--kind=' + kind, file_path, stdout=out, verbosity=0) - self.assertEqual(out.getvalue(), '') + call_command("importcsv", "--kind=" + kind, file_path, stdout=out, verbosity=0) + self.assertEqual(out.getvalue(), "") # Ensure the import is logged correctly import_log = ImportLog.objects.get(export_kind=kind) @@ -69,103 +68,103 @@ def test_importcsv_with_kind(self): self.assertEqual(import_log.is_incremental, True) self.assertEqual(import_log.error, ImportLog.SUCCESS) self.assertGreater(import_log.num_modified, 0) - self.assertNotEqual(import_log.message, '') + self.assertNotEqual(import_log.message, "") def test_import_csv_user(self): # Ensure that the test user did not exist beforehand, in the test database - self.assertEqual(Account.objects.filter(xorg_id='louis.vaneau.1829').count(), 0) + self.assertEqual(Account.objects.filter(xorg_id="louis.vaneau.1829").count(), 0) self.assertEqual(Account.objects.filter(af_id=1).count(), 0) # Import the test user and test the result - call_command('importcsv', TEST_CSV_PATHS['users'], verbosity=0) + call_command("importcsv", TEST_CSV_PATHS["users"], verbosity=0) user = Account.objects.get(af_id=1) self.assertIsNotNone(user) self.assertEqual(user.af_id, 1) - self.assertEqual(user.ax_id, '18290001') - self.assertEqual(user.first_name, 'Louis') - self.assertEqual(user.last_name, 'Vaneau') - self.assertEqual(user.common_name, 'Vaneau') - self.assertEqual(user.civility, 'M') + self.assertEqual(user.ax_id, "18290001") + self.assertEqual(user.first_name, "Louis") + self.assertEqual(user.last_name, "Vaneau") + self.assertEqual(user.common_name, "Vaneau") + self.assertEqual(user.civility, "M") self.assertEqual(user.birthdate, datetime.date(1811, 3, 27)) - self.assertEqual(user.address_1, 'rue de Babylone') - self.assertEqual(user.address_2, '') - self.assertEqual(user.address_3, '') - self.assertEqual(user.address_4, '') - self.assertEqual(user.address_postcode, '75007') - self.assertEqual(user.address_city, 'PARIS') - self.assertEqual(user.address_state, '') - self.assertEqual(user.address_country, 'FR') + self.assertEqual(user.address_1, "rue de Babylone") + self.assertEqual(user.address_2, "") + self.assertEqual(user.address_3, "") + self.assertEqual(user.address_4, "") + self.assertEqual(user.address_postcode, "75007") + self.assertEqual(user.address_city, "PARIS") + self.assertEqual(user.address_state, "") + self.assertEqual(user.address_country, "FR") self.assertEqual(user.address_npai, False) - self.assertEqual(user.phone_personnal, '06 00 00 00 00') - self.assertEqual(user.phone_mobile, '06 00 00 00 00') - self.assertEqual(user.email_1, 'louis.vaneau.1829@polytechnique.org') - self.assertEqual(user.email_2, 'louis.vaneau.1829+ax@polytechnique.org') - self.assertEqual(user.nationality, 'France') - self.assertEqual(user.nationality_2, '') - self.assertEqual(user.nationality_3, '') + self.assertEqual(user.phone_personnal, "06 00 00 00 00") + self.assertEqual(user.phone_mobile, "06 00 00 00 00") + self.assertEqual(user.email_1, "louis.vaneau.1829@polytechnique.org") + self.assertEqual(user.email_2, "louis.vaneau.1829+ax@polytechnique.org") + self.assertEqual(user.nationality, "France") + self.assertEqual(user.nationality_2, "") + self.assertEqual(user.nationality_3, "") self.assertEqual(user.dead, True) self.assertEqual(user.deathdate, datetime.date(1830, 7, 29)) - self.assertEqual(user.dead_for_france, '') + self.assertEqual(user.dead_for_france, "") self.assertEqual(user.user_kind, 1) - self.assertEqual(user.additional_roles, '') - self.assertEqual(user.xorg_id, 'louis.vaneau.1829') - self.assertEqual(user.school_id, '18290001') - self.assertEqual(user.admission_path, '') - self.assertEqual(user.cursus_domain, '') - self.assertEqual(user.cursus_name, '') - self.assertEqual(user.corps_current, 'Aucun (anc. démissionnaire)') - self.assertEqual(user.corps_origin, '') - self.assertEqual(user.corps_grade, 'Aucun') - self.assertEqual(user.nickname, '') - self.assertEqual(user.sport_section, 'Escrime') - self.assertEqual(user.binets, 'Binet Escrime') - self.assertEqual(user.mail_reception, '') - self.assertEqual(user.newsletter_inscriptions, 'Lettre mensuelle de Polytechnique.org,Lettre de la communauté') - self.assertEqual(user.profile_picture_url, 'https://ax.polytechnique.org/medias/profile/42.jpeg') + self.assertEqual(user.additional_roles, "") + self.assertEqual(user.xorg_id, "louis.vaneau.1829") + self.assertEqual(user.school_id, "18290001") + self.assertEqual(user.admission_path, "") + self.assertEqual(user.cursus_domain, "") + self.assertEqual(user.cursus_name, "") + self.assertEqual(user.corps_current, "Aucun (anc. démissionnaire)") + self.assertEqual(user.corps_origin, "") + self.assertEqual(user.corps_grade, "Aucun") + self.assertEqual(user.nickname, "") + self.assertEqual(user.sport_section, "Escrime") + self.assertEqual(user.binets, "Binet Escrime") + self.assertEqual(user.mail_reception, "") + self.assertEqual(user.newsletter_inscriptions, "Lettre mensuelle de Polytechnique.org,Lettre de la communauté") + self.assertEqual(user.profile_picture_url, "https://ax.polytechnique.org/medias/profile/42.jpeg") self.assertEqual(user.last_update, datetime.date(2001, 2, 3)) self.assertEqual(user.deleted_since, None) # Import its studies and test the result self.assertEqual(user.degrees.count(), 0) - call_command('importcsv', TEST_CSV_PATHS['userdegrees'], verbosity=0) + call_command("importcsv", TEST_CSV_PATHS["userdegrees"], verbosity=0) user.refresh_from_db() self.assertEqual(user.degrees.count(), 1) user_degree = user.degrees.all()[0] self.assertEqual(user_degree.account, user) - self.assertEqual(user_degree.diploma_reference, '1') + self.assertEqual(user_degree.diploma_reference, "1") self.assertEqual(user_degree.diplomed, True) self.assertEqual(user_degree.diplomation_date, datetime.date(1829, 1, 1)) - self.assertEqual(user_degree.cycle, '') - self.assertEqual(user_degree.domain, '') - self.assertEqual(user_degree.name, 'Ingénieur') + self.assertEqual(user_degree.cycle, "") + self.assertEqual(user_degree.domain, "") + self.assertEqual(user_degree.name, "Ingénieur") self.assertEqual(user_degree.last_update, datetime.date(2001, 2, 3)) # Import its jobs and test the result self.assertEqual(user.jobs.count(), 0) - call_command('importcsv', TEST_CSV_PATHS['userjobs'], verbosity=0) + call_command("importcsv", TEST_CSV_PATHS["userjobs"], verbosity=0) user.refresh_from_db() self.assertEqual(user.jobs.count(), 2) user_job = user.jobs.all()[0] self.assertEqual(user_job.account, user) - self.assertEqual(user_job.title, 'Emploi numéro 1') - self.assertEqual(user_job.role, 'Fonction 1') - self.assertEqual(user_job.company_name, 'Une Entreprise Exemplaire') - self.assertEqual(user_job.address_1, 'rue Quelconque') - self.assertEqual(user_job.address_2, '') - self.assertEqual(user_job.address_3, '') - self.assertEqual(user_job.address_4, '') - self.assertEqual(user_job.address_postcode, '75000') - self.assertEqual(user_job.address_city, 'Paris') - self.assertEqual(user_job.address_country, 'FR') + self.assertEqual(user_job.title, "Emploi numéro 1") + self.assertEqual(user_job.role, "Fonction 1") + self.assertEqual(user_job.company_name, "Une Entreprise Exemplaire") + self.assertEqual(user_job.address_1, "rue Quelconque") + self.assertEqual(user_job.address_2, "") + self.assertEqual(user_job.address_3, "") + self.assertEqual(user_job.address_4, "") + self.assertEqual(user_job.address_postcode, "75000") + self.assertEqual(user_job.address_city, "Paris") + self.assertEqual(user_job.address_country, "FR") self.assertEqual(user_job.phone_indicator, None) - self.assertEqual(user_job.phone_number, '01 00 00 00 00') + self.assertEqual(user_job.phone_number, "01 00 00 00 00") self.assertEqual(user_job.mobile_phone_indicator, None) - self.assertEqual(user_job.mobile_phone_number, '06 00 00 00 00') - self.assertEqual(user_job.fax, '') - self.assertEqual(user_job.email, 'louis.vaneau@example.org') + self.assertEqual(user_job.mobile_phone_number, "06 00 00 00 00") + self.assertEqual(user_job.fax, "") + self.assertEqual(user_job.email, "louis.vaneau@example.org") self.assertEqual(user_job.start_date, datetime.date(1820, 1, 1)) self.assertEqual(user_job.end_date, datetime.date(1821, 1, 1)) - self.assertEqual(user_job.contract_kind, 'CDI') + self.assertEqual(user_job.contract_kind, "CDI") self.assertEqual(user_job.current, True) self.assertEqual(user_job.creator_of_company, False) self.assertEqual(user_job.buyer_of_company, False) @@ -173,25 +172,25 @@ def test_import_csv_user(self): user_job = user.jobs.all()[1] self.assertEqual(user_job.account, user) - self.assertEqual(user_job.title, 'Emploi numéro 2') - self.assertEqual(user_job.role, 'Fonction 2') - self.assertEqual(user_job.company_name, 'Une Seconde Entreprise Exemplaire') - self.assertEqual(user_job.address_1, 'avenue Quelconque') - self.assertEqual(user_job.address_2, '') - self.assertEqual(user_job.address_3, '') - self.assertEqual(user_job.address_4, '') - self.assertEqual(user_job.address_postcode, '75000') - self.assertEqual(user_job.address_city, 'Paris') - self.assertEqual(user_job.address_country, 'FR') + self.assertEqual(user_job.title, "Emploi numéro 2") + self.assertEqual(user_job.role, "Fonction 2") + self.assertEqual(user_job.company_name, "Une Seconde Entreprise Exemplaire") + self.assertEqual(user_job.address_1, "avenue Quelconque") + self.assertEqual(user_job.address_2, "") + self.assertEqual(user_job.address_3, "") + self.assertEqual(user_job.address_4, "") + self.assertEqual(user_job.address_postcode, "75000") + self.assertEqual(user_job.address_city, "Paris") + self.assertEqual(user_job.address_country, "FR") self.assertEqual(user_job.phone_indicator, 33) - self.assertEqual(user_job.phone_number, '01 00 00 00 00') + self.assertEqual(user_job.phone_number, "01 00 00 00 00") self.assertEqual(user_job.mobile_phone_indicator, 33) - self.assertEqual(user_job.mobile_phone_number, '06 00 00 00 00') - self.assertEqual(user_job.fax, '') - self.assertEqual(user_job.email, 'louis.vaneau@example.net') + self.assertEqual(user_job.mobile_phone_number, "06 00 00 00 00") + self.assertEqual(user_job.fax, "") + self.assertEqual(user_job.email, "louis.vaneau@example.net") self.assertEqual(user_job.start_date, datetime.date(1819, 1, 1)) self.assertEqual(user_job.end_date, datetime.date(1820, 1, 1)) - self.assertEqual(user_job.contract_kind, 'CDI') + self.assertEqual(user_job.contract_kind, "CDI") self.assertEqual(user_job.current, False) self.assertEqual(user_job.creator_of_company, False) self.assertEqual(user_job.buyer_of_company, False) @@ -199,31 +198,31 @@ def test_import_csv_user(self): def test_import_csv_group(self): # Ensure that the test user and group did not exist beforehand, in the test database - self.assertEqual(Account.objects.filter(xorg_id='louis.vaneau.1829').count(), 0) + self.assertEqual(Account.objects.filter(xorg_id="louis.vaneau.1829").count(), 0) self.assertEqual(Account.objects.filter(af_id=1).count(), 0) self.assertEqual(Group.objects.filter(af_id=1).count(), 0) - call_command('importcsv', TEST_CSV_PATHS['users'], verbosity=0) - call_command('importcsv', TEST_CSV_PATHS['groups'], verbosity=0) + call_command("importcsv", TEST_CSV_PATHS["users"], verbosity=0) + call_command("importcsv", TEST_CSV_PATHS["groups"], verbosity=0) user = Account.objects.get(af_id=1) self.assertIsNotNone(user) group = Group.objects.get(af_id=1) self.assertIsNotNone(group) self.assertEqual(group.af_id, 1) - self.assertEqual(group.ax_id, 'AF_1') - self.assertEqual(group.url, 'https://ax.polytechnique.org/group/GroupeUnit%C3%A9/1') - self.assertEqual(group.name, 'Groupe Unité') - self.assertEqual(group.category, 'Acteurs de la communauté') + self.assertEqual(group.ax_id, "AF_1") + self.assertEqual(group.url, "https://ax.polytechnique.org/group/GroupeUnit%C3%A9/1") + self.assertEqual(group.name, "Groupe Unité") + self.assertEqual(group.category, "Acteurs de la communauté") self.assertEqual(group.last_update, datetime.date(2001, 2, 3)) # Import group memberships self.assertEqual(user.group_memberships.count(), 0) self.assertEqual(group.memberships.count(), 0) - call_command('importcsv', TEST_CSV_PATHS['groupmembers'], verbosity=0) + call_command("importcsv", TEST_CSV_PATHS["groupmembers"], verbosity=0) self.assertEqual(user.group_memberships.count(), 2) self.assertEqual(group.memberships.count(), 1) membership = user.group_memberships.get(group=group) self.assertEqual(membership.account, user) self.assertEqual(membership.group, group) - self.assertEqual(membership.role, 'member') + self.assertEqual(membership.role, "member") self.assertEqual(membership.last_update, datetime.date(2001, 2, 3)) diff --git a/tests/test_views.py b/tests/test_views.py index 0765659..98b7a84 100644 --- a/tests/test_views.py +++ b/tests/test_views.py @@ -8,13 +8,13 @@ class ViewTests(TestCase): # Views which are publicy accessible PUBLIC_VIEW_IDS = ( - 'index', - 'robots', + "index", + "robots", ) # Views which need an authenticated user LOGIN_REQUIRED_VIEW_IDS = ( - 'admin:index', - 'issues', + "admin:index", + "issues", ) def test_know_all_views(self): @@ -29,7 +29,7 @@ def test_know_all_views(self): except AttributeError: pass # admin:index is special, because it comes from an inclusion - known_views.remove('admin:index') + known_views.remove("admin:index") # Ensure that every view in the local lists exist self.assertEqual(set(), known_views, "stray view IDs in tests") @@ -46,29 +46,29 @@ def test_login_required_views_forbidden(self): c = Client() for url_id in self.LOGIN_REQUIRED_VIEW_IDS: resp = c.get(reverse(url_id)) - self.assertEqual(302, resp.status_code, - "unexpected HTTP response code for URL %s" % url_id) - self.assertTrue(resp['Location'].startswith('/admin/login/?'), - "unexpected Location header: %r" % resp['Location']) + self.assertEqual(302, resp.status_code, "unexpected HTTP response code for URL %s" % url_id) + self.assertTrue( + resp["Location"].startswith("/admin/login/?"), "unexpected Location header: %r" % resp["Location"] + ) def test_login_required_views_success(self): """Test accessing login-required views while being logged in""" # Create a dummy super user User.objects.create_superuser( - username='superuser', - email='superuser@localhost.localdomain', - password='A random insecure password', + username="superuser", + email="superuser@localhost.localdomain", + password="A random insecure password", ) for url_id in self.LOGIN_REQUIRED_VIEW_IDS: c = Client() - self.assertTrue(c.login(username='superuser', password='A random insecure password')) + self.assertTrue(c.login(username="superuser", password="A random insecure password")) resp = c.get(reverse(url_id)) if resp.status_code == 302: - self.assertFalse(resp['Location'].startswith(('/accounts/login/?', '/auth-groupex-login?')), - "unexpected login-Location: %r" % resp['Location']) - elif url_id == 'auth-groupex': - self.assertEqual(400, resp.status_code, - "unexpected HTTP response code for URL %s" % url_id) + self.assertFalse( + resp["Location"].startswith(("/accounts/login/?", "/auth-groupex-login?")), + "unexpected login-Location: %r" % resp["Location"], + ) + elif url_id == "auth-groupex": + self.assertEqual(400, resp.status_code, "unexpected HTTP response code for URL %s" % url_id) else: - self.assertEqual(200, resp.status_code, - "unexpected HTTP response code for URL %s" % url_id) + self.assertEqual(200, resp.status_code, "unexpected HTTP response code for URL %s" % url_id) diff --git a/tox.ini b/tox.ini index 17af050..0669c0e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,40 +1,33 @@ [tox] envlist = - py34-django{111,20} - py{35,36,37}-django{111,20,21,22} + py{311,312,313}-django5 lint dev prodsettings - toxworkdir = {env:TOX_WORKDIR:.tox} [testenv] deps = -rrequirements_dev.txt - django111: Django>=1.11,<1.12 - django20: Django>=2.0,<2.1 - django21: Django>=2.1,<2.2 - django22: Django>=2.2,<2.3 - -whitelist_externals = make + django5: Django>=5.1,<6 +allowlist_externals = make commands = make test [testenv:lint] deps = -rrequirements_dev.txt - -whitelist_externals = make +allowlist_externals = make commands = make lint [testenv:dev] -whitelist_externals = make +allowlist_externals = make commands = make update make make createdb [testenv:prodsettings] -whitelist_externals = make +allowlist_externals = make setenv = XORGDATA_APP_MODE = prod XORGDATA_APP_SECRET_KEY = this is a long key to pass SECRET_KEY length check diff --git a/xorgdata/__init__.py b/xorgdata/__init__.py index df9144c..d3ec452 100644 --- a/xorgdata/__init__.py +++ b/xorgdata/__init__.py @@ -1 +1 @@ -__version__ = '0.1.1' +__version__ = "0.2.0" diff --git a/xorgdata/alumnforce/admin.py b/xorgdata/alumnforce/admin.py index fd93722..a4cef07 100644 --- a/xorgdata/alumnforce/admin.py +++ b/xorgdata/alumnforce/admin.py @@ -5,7 +5,7 @@ from django.db.models import Count, Q from django.urls import reverse from django.utils.html import format_html -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from . import models @@ -23,31 +23,34 @@ class ProfessionnalInformationInline(admin.StackedInline): class GroupMembershipInline(admin.TabularInline): model = models.GroupMembership extra = 0 - readonly_fields = ('link_account', 'link_group') - fields = ('link_account', 'link_group', 'role') + readonly_fields = ("link_account", "link_group") + fields = ("link_account", "link_group", "role") def link_account(self, obj): obj_url = reverse( - 'admin:%s_%s_change' % (obj.account._meta.app_label, obj.account._meta.model_name), - args=(obj.account.af_id, )) + "admin:%s_%s_change" % (obj.account._meta.app_label, obj.account._meta.model_name), + args=(obj.account.af_id,), + ) return format_html('{}', obj_url, obj.account) + link_account.short_description = _("account") def link_group(self, obj): obj_url = reverse( - 'admin:%s_%s_change' % (obj.group._meta.app_label, obj.group._meta.model_name), - args=(obj.group.af_id, )) + "admin:%s_%s_change" % (obj.group._meta.app_label, obj.group._meta.model_name), args=(obj.group.af_id,) + ) return format_html('{}', obj_url, obj.group) + link_group.short_description = _("group") @admin.register(models.Account) class AccountAdmin(admin.ModelAdmin): - search_fields = ('ax_id', 'xorg_id', 'first_name', 'last_name', 'common_name') - list_display = ('af_id', 'ax_id', 'xorg_id', 'first_name', 'last_name', 'deleted_since') - list_display_links = ('af_id', 'ax_id', 'xorg_id', 'first_name', 'last_name') - readonly_fields = ('kind_desc', 'roles_desc', 'alumnforce_profile_url') - ordering = ('-ax_id', 'xorg_id', 'af_id') + search_fields = ("ax_id", "xorg_id", "first_name", "last_name", "common_name") + list_display = ("af_id", "ax_id", "xorg_id", "first_name", "last_name", "deleted_since") + list_display_links = ("af_id", "ax_id", "xorg_id", "first_name", "last_name") + readonly_fields = ("kind_desc", "roles_desc", "alumnforce_profile_url") + ordering = ("-ax_id", "xorg_id", "af_id") inlines = [ AcademicInformationInline, @@ -57,12 +60,14 @@ class AccountAdmin(admin.ModelAdmin): def kind_desc(self, obj): """Get the description of account kind""" - return "{} [{}]".format(models.Account.KINDS.get(obj.user_kind, '?'), obj.user_kind) + return "{} [{}]".format(models.Account.KINDS.get(obj.user_kind, "?"), obj.user_kind) + kind_desc.short_description = _("Account kind") def roles_desc(self, obj): """Get the description of account additional roles""" - return ', '.join("{} [{}]".format(models.Account.ROLES.get(r, '?'), r) for r in obj.get_additional_roles()) + return ", ".join("{} [{}]".format(models.Account.ROLES.get(r, "?"), r) for r in obj.get_additional_roles()) + roles_desc.short_description = _("Additional roles") def alumnforce_profile_url(self, obj): @@ -71,11 +76,11 @@ def alumnforce_profile_url(self, obj): @admin.register(models.Group) class GroupAdmin(admin.ModelAdmin): - search_fields = ('af_id', 'name', 'category') - list_display = ('af_id', 'ax_id', 'name', 'count_members', 'category', 'url') - list_display_links = ('af_id', 'ax_id', 'name') - readonly_fields = ('url_link', ) - ordering = ('name', 'af_id') + search_fields = ("af_id", "name", "category") + list_display = ("af_id", "ax_id", "name", "count_members", "category", "url") + list_display_links = ("af_id", "ax_id", "name") + readonly_fields = ("url_link",) + ordering = ("name", "af_id") inlines = [ GroupMembershipInline, ] @@ -83,28 +88,30 @@ class GroupAdmin(admin.ModelAdmin): def get_queryset(self, request): queryset = super().get_queryset(request) queryset = queryset.annotate( - _members_count=Count("memberships", - filter=Q(memberships__role__in=models.GroupMembership.IN_GROUP_ROLES), - distinct=True), + _members_count=Count( + "memberships", filter=Q(memberships__role__in=models.GroupMembership.IN_GROUP_ROLES), distinct=True + ), ) return queryset def count_members(self, obj): return obj._members_count + count_members.short_description = _("members") def url_link(self, obj): return format_html('{0}', obj.url) + url_link.short_description = _("URL link") @admin.register(models.ImportLog) class ImportLogAdmin(admin.ModelAdmin): - list_display = ('date', 'export_kind', 'is_incremental', 'error', 'num_modified', 'message') - ordering = ('-date', 'export_kind') + list_display = ("date", "export_kind", "is_incremental", "error", "num_modified", "message") + ordering = ("-date", "export_kind") @admin.register(models.ExportLog) class ExportLogAdmin(admin.ModelAdmin): - list_display = ('date', 'export_kind', 'error', 'num_items', 'message') - ordering = ('-date', 'export_kind') + list_display = ("date", "export_kind", "error", "num_items", "message") + ordering = ("-date", "export_kind") diff --git a/xorgdata/alumnforce/apps.py b/xorgdata/alumnforce/apps.py index dfd3995..9e248bd 100644 --- a/xorgdata/alumnforce/apps.py +++ b/xorgdata/alumnforce/apps.py @@ -2,4 +2,4 @@ class AlumnforceConfig(AppConfig): - name = 'alumnforce' + name = "xorgdata.alumnforce" diff --git a/xorgdata/alumnforce/full_export/convert_csv_to_json.py b/xorgdata/alumnforce/full_export/convert_csv_to_json.py index bad727f..1f0dd12 100755 --- a/xorgdata/alumnforce/full_export/convert_csv_to_json.py +++ b/xorgdata/alumnforce/full_export/convert_csv_to_json.py @@ -23,6 +23,7 @@ ./convert_csv_to_json.py -u export-users-20180314-159265.csv | \ jq '.[] | select(.email and (.email | any(contains("@alumnforce"))))' """ + import argparse import sys @@ -31,14 +32,15 @@ def main(): parser = argparse.ArgumentParser(description="Convert AF CSV to JSON") - parser.add_argument('file', nargs='?', - help="CSV file to read (or standard input)") - parser.add_argument('-k', '--keep-empty', action='store_true', - help="keep empty fields instead of dropping them") - parser.add_argument('-o', '--output', type=str, - help="JSON file to write (or standard output)") - parser.add_argument('-u', '--utf8', action='store_true', - help="write unicode strings as UTF-8 instead of escaping characters with \\u") + parser.add_argument("file", nargs="?", help="CSV file to read (or standard input)") + parser.add_argument("-k", "--keep-empty", action="store_true", help="keep empty fields instead of dropping them") + parser.add_argument("-o", "--output", type=str, help="JSON file to write (or standard output)") + parser.add_argument( + "-u", + "--utf8", + action="store_true", + help="write unicode strings as UTF-8 instead of escaping characters with \\u", + ) args = parser.parse_args() if args.file: @@ -46,12 +48,12 @@ def main(): else: data = AlumnForceDataC2J.import_csv_stream(sys.stdin.buffer, args.keep_empty) - if args.output and args.output != '-': - with open(args.output, 'w') as fjson: + if args.output and args.output != "-": + with open(args.output, "w") as fjson: data.json_dump(fjson, indent=2, ensure_ascii=not args.utf8) else: data.json_dump(sys.stdout, indent=2, ensure_ascii=not args.utf8) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/xorgdata/alumnforce/full_export/convert_json_to_csv.py b/xorgdata/alumnforce/full_export/convert_json_to_csv.py index eeedc4e..b8fd6d4 100755 --- a/xorgdata/alumnforce/full_export/convert_json_to_csv.py +++ b/xorgdata/alumnforce/full_export/convert_json_to_csv.py @@ -4,6 +4,7 @@ This is the reciprocal of convert_csv_to_json.py. """ + import argparse import sys @@ -12,10 +13,8 @@ def main(): parser = argparse.ArgumentParser(description="Convert AF CSV to JSON") - parser.add_argument('file', nargs='?', - help="JSON file to read (or standard input)") - parser.add_argument('-o', '--output', type=str, - help="CSV file to write (or standard output)") + parser.add_argument("file", nargs="?", help="JSON file to read (or standard input)") + parser.add_argument("-o", "--output", type=str, help="CSV file to write (or standard output)") args = parser.parse_args() if args.file: @@ -23,12 +22,12 @@ def main(): else: data = AlumnForceDataJ2C.import_json_stream(sys.stdin) - if args.output and args.output != '-': - with open(args.output, 'w') as fcsv: + if args.output and args.output != "-": + with open(args.output, "w") as fcsv: data.csv_dump(fcsv) else: data.csv_dump(sys.stdout) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/xorgdata/alumnforce/full_export/lib/converters.py b/xorgdata/alumnforce/full_export/lib/converters.py index 39a27cb..b3bf2b6 100644 --- a/xorgdata/alumnforce/full_export/lib/converters.py +++ b/xorgdata/alumnforce/full_export/lib/converters.py @@ -5,7 +5,6 @@ from .csv_format import ALUMNFORCE_FIELDS - CSV_TO_JSON_FIELDS = dict((x[0], (x[1], x[2])) for x in ALUMNFORCE_FIELDS) assert len(ALUMNFORCE_FIELDS) == len(CSV_TO_JSON_FIELDS) @@ -15,6 +14,7 @@ class AlumnForceDataC2J(object): """Data extracted from AlumnForce website""" + def __init__(self): self.fields = None self.content = [] @@ -22,14 +22,14 @@ def __init__(self): @classmethod def import_csv_file(cls, csv_file_path, keep_empty=False): """Create AlumnForce data from a CSV file""" - with open(csv_file_path, 'r', encoding='iso-8859-15') as csv_stream: + with open(csv_file_path, "r", encoding="iso-8859-15") as csv_stream: return cls.import_csv_stream(csv_stream, keep_empty) @classmethod def import_csv_stream(cls, csv_file, keep_empty=False): """Create AlumnForce data from a CSV stream""" data = cls() - reader = csv.reader(csv_file, delimiter=',', quotechar='"', escapechar='\\', strict=True) + reader = csv.reader(csv_file, delimiter=",", quotechar='"', escapechar="\\", strict=True) for row in reader: if reader.line_num == 1: data.set_fields_from_csv(row) @@ -52,13 +52,13 @@ def decode_csv_row(self, csv_row, keep_empty): """Decode a row of the CSV file""" if len(csv_row) != len(self.fields): raise ValueError( - "CSV row of length %d not the length of fields (%d): %r" % ( - len(csv_row), len(self.fields), csv_row)) + "CSV row of length %d not the length of fields (%d): %r" % (len(csv_row), len(self.fields), csv_row) + ) data_row = collections.OrderedDict() for field_name_type, value in zip(self.fields.items(), csv_row): field_name, field_type = field_name_type - if not keep_empty and value == '': + if not keep_empty and value == "": continue # Convert the value to the field type @@ -72,8 +72,8 @@ def decode_csv_row(self, csv_row, keep_empty): # Decode name parts data_directory = data_row - while '.' in field_name: - dir_name, field_name = field_name.split('.', 1) + while "." in field_name: + dir_name, field_name = field_name.split(".", 1) if dir_name not in data_directory: data_directory[dir_name] = collections.OrderedDict() data_directory = data_directory[dir_name] @@ -87,6 +87,7 @@ def json_dump(self, fp, **kwargs): class AlumnForceDataJ2C(object): """Data extracted from a JSON to produce data importted on AlumnForce website""" + def __init__(self): self.fields = set() # content is a list of dicts "json field"->value @@ -95,7 +96,7 @@ def __init__(self): @classmethod def import_json_file(cls, json_file_path): """Create AlumnForce data from a JSON file""" - with open(json_file_path, 'r') as json_stream: + with open(json_file_path, "r") as json_stream: return cls.import_json_stream(json_stream) @classmethod @@ -123,7 +124,7 @@ def flatten_json_fields(cls, json_record, prefixkey=None): result.append((fullkey, value)) elif isinstance(value, dict): # sub-dict - result += cls.flatten_json_fields(value, fullkey + '.') + result += cls.flatten_json_fields(value, fullkey + ".") else: raise ValueError("Unknown json field %r" % fullkey) return result @@ -132,7 +133,7 @@ def csv_dump(self, csv_file, **kwargs): """Dump all the CSV data""" # Sort the fields by their rank in ALUMNFORCE_FIELDS columns = sorted(self.fields, key=lambda f: JSON_TO_CSV_FIELDS[f][2]) - writer = csv.writer(csv_file, delimiter=',', quotechar='"', escapechar='\\', quoting=csv.QUOTE_MINIMAL) + writer = csv.writer(csv_file, delimiter=",", quotechar='"', escapechar="\\", quoting=csv.QUOTE_MINIMAL) writer.writerow((JSON_TO_CSV_FIELDS[f][0] for f in columns)) for row in self.content: writer.writerow(row.get(f) for f in columns) diff --git a/xorgdata/alumnforce/full_export/lib/csv_format.py b/xorgdata/alumnforce/full_export/lib/csv_format.py index dd31f56..919c431 100644 --- a/xorgdata/alumnforce/full_export/lib/csv_format.py +++ b/xorgdata/alumnforce/full_export/lib/csv_format.py @@ -7,198 +7,234 @@ class BoolType(object): """A boolean in AlumnForce CSV""" + @staticmethod def encode(value): if value is None: - return '' - return '1' if value else '0' + return "" + return "1" if value else "0" @staticmethod def decode(value): - if value == '0': + if value == "0": return False - elif value == '1': + elif value == "1": return True - elif value == '': + elif value == "": return None raise ValueError() class YesNoBoolType(object): """A boolean in AlumnForce CSV encoded as "Oui" and "Non" """ + @staticmethod def encode(value): if value is None: - return '' - return 'Oui' if value else 'Non' + return "" + return "Oui" if value else "Non" @staticmethod def decode(value): - if value == 'Non': + if value == "Non": return False - elif value == 'Oui': + elif value == "Oui": return True - elif value == '': + elif value == "": return None raise ValueError() class CommaListType(object): """A list of values separated by commas""" + @staticmethod def encode(value): if not value: - return '' - return ','.join(value) + return "" + return ",".join(value) @staticmethod def decode(value): if not value: return None - return value.split(',') + return value.split(",") class CommaSpaceListType(object): """A list of values separated by commas""" + @staticmethod def encode(value): if not value: - return '' - return ', '.join(value) + return "" + return ", ".join(value) @staticmethod def decode(value): if not value: return None - return value.split(', ') + return value.split(", ") # CSV and JSON field names with the data type ("None" type is unicode string) # Use "." for sub-object fields in JSON ALUMNFORCE_FIELDS = ( - ('Identifiant (AlumnForce)', 'id_af', None), - ('Identifiant (École)', 'id_ax', None), - ('Prénom', 'first_name', None), - ('Nom d\'état-civil', 'last_name', None), - ('Nom d\'usage', 'usage_name', None), - ('Civilité (Mme / Mlle / M.)', 'civility', None), - ('Date de naissance', 'birth_date', None), - ('Date de création de compte', 'account_creation_date', None), - ('Date d\'activation de compte', 'account_activation_date', None), - ('Date de validation des CGU', 'cgu_validation_date', None), - ('Adresse personnelle - Ligne 1', 'personal.address.line_1', None), - ('Adresse personnelle - Ligne 2', 'personal.address.line_2', None), - ('Adresse personnelle - Ligne 3', 'personal.address.line_3', None), - ('Adresse personnelle - Ligne 4', 'personal.address.line_4', None), - ('Adresse personnelle - Région', 'personal.address.region', None), - ('Adresse personnelle - Ville', 'personal.address.city', None), - ('Adresse personnelle - État', 'personal.address.state', None), - ('Adresse personnelle - Code postal', 'personal.address.code', None), - ('Adresse personnelle - Cedex', 'personal.address.cedex', None), - ('Adresse personnelle - Pays (ISO)', 'personal.address.country', None), - ('Adresse personnelle - NPAI (Oui [1] / Non [0])', 'personal.address.bounced', BoolType), - ('Téléphone fixe personnel', 'personal.fix_phone', None), - ('Téléphone mobile personnel', 'personal.cell_phone', None), - ('Email personnel 1', 'email.personal_1', None), - ('Email personnel 2', 'email.personal_2', None), - ('Nationalité', 'nationality', None), - ('Situation matrimoniale', 'marital_status', None), - ('Membre décédé (Oui [1] / Non [0])', 'is_dead', BoolType), - ("Statut (Non enregistré [0] / En cours d'activation [1] / Compte bloqué [2] / Compte activé [3] / Clé d'activation envoyée [4] / En attente de validation [5] / Ne souhaite pas activer son compte [6] / Compte expiré [8])", 'account_status', None), # noqa - ('Compte activé (Oui [1] / Non [0])', 'is_activated', BoolType), - ("Type utilisateur (Diplômé(e) [1] / Personnel de l'association [3] / Élève / étudiant(e) [5] / Visiteur [7] / Membre associé [9] / Veuves/Veufs [10])", 'user_kind', None), # noqa - ('Rôle supplémentaire (Visiteur [1] / Administrateur total [3] / Diplômé [4] / Cotisant [5] / Élève et étudiant [7] / Abonné [17] / Membre associé [19] / Administrateur contenu [21] / Administrateur comptable [22] / Veuves/Veufs [26])', 'roles', CommaListType), # noqa - ('Forcer le statut de cotisant (Oui [1] / Non [0])', 'force_contributor', BoolType), - ('Référence du diplôme', 'school.degree_ref', None), - ('Ecole', 'school.name', None), - ('Filière (L/M/D)', 'school.stream', None), - ('Spécialisation', 'speciality', None), - ('Parcours', 'curriculum', None), - ("Niveau d'étude / Nombre d'années post bac", 'study_level', None), - ('Mode', 'mode', None), - ('A obtenu son diplôme ? (Oui [1] / Non [0])', 'school.has_graduated', BoolType), - ('Date de promotion', 'school.graduation_date', None), - ("Date d'intégration", 'school.entry_date', None), - ('Publier données académiques', 'school.publish_academic', BoolType), - ('Situation actuelle', 'work.current_situation', None), - ('Poste actuel', 'work.current_job', None), - ('Code Fontion / Métier', 'work.job_code', None), - ('Fonction / Métier', 'work.function', None), - ('Niveau du poste', 'work.job_level', None), - ('Entreprise - Nom', 'work.company.name', None), - ("Entreprise - Secteur d'activité", 'work.company.sector', None), - ('Entreprise - Code SIRET', 'work.company.siret', None), - ('Entreprise - Site internet', 'work.company.website', None), - ('Adresse professionnelle - Ligne 1', 'work.company.address.line_1', None), - ('Adresse professionnelle - Ligne 2', 'work.company.address.line_2', None), - ('Adresse professionnelle - Ligne 3', 'work.company.address.line_3', None), - ('Adresse professionnelle - Ligne 4', 'work.company.address.line_4', None), - ('Adresse professionnelle - Région', 'work.company.address.region', None), - ('Adresse professionnelle - Ville', 'work.company.address.city', None), - ('Adresse professionnelle - État', 'work.company.address.country', None), - ('Adresse professionnelle - Code postal', 'work.company.address.code', None), - ('Adresse professionnelle - Cedex', 'work.company.address.cedex', None), - ('Adresse professionnelle - Pays (ISO)', 'work.company.address.iso', None), - ('Téléphone fixe professionnel', 'work.company.fix_phone', None), - ('Téléphone mobile professionnel', 'work.company.cell_phone', None), - ('Fax professionnel', 'work.company.fax', None), - ('Email professionnel', 'email.professional', None), - ("Début de l'expérience", 'work.company.begin', None), - ("Fin de l'expérience", 'work.company.end', None), - ('Type de contrat', 'work.company.contract_kind', None), - ('Salaire réel', 'work.company.wage', None), - ('Tranche de salaire', 'work.company.wage_bracket', None), - ("Je souhaite contribuer à la vie de l'Association: CA, Colloque, PDX, Bal de l'X, Grand Magnan ... (Oui [1] / Non [0])", 'wish.ax', BoolType), # noqa - ('Je suis prêt à aider des camarades en transition professionnelle, ou porteurs de projet (Oui [1] / Non [0])', 'wish.help', BoolType), # noqa - ('Je suis intéressé par les programmes de parrainage et de mentoring (Oui [1] / Non [0])', 'wish.mentoring', BoolType), # noqa - ('Je suis prêt à donner un peu de temps pour aider des camarades, ou familles de camarades, en difficulté via la Caisse de Solidarité (Oui [1] / Non [0])', 'wish.solidarity', BoolType), # noqa - ("J'accepte de recevoir les mails de l'Association (Oui [1] / Non [0])", 'wish.ax_emails', BoolType), - ("Je peux intervenir lors d'événements, pour animer un atelier, participer à une table ronde ou donner une conférence (Oui [1] / Non [0])", 'wish.animate', BoolType), # noqa - ("J'accepte de figurer dans l'annuaire papier (Oui [1] / Non [0])", 'wish.in_directory', BoolType), - ("Je souhaite recevoir l'annuaire papier (Oui [1] / Non [0])", 'wish.receive_directory', BoolType), - ("Email d'identification", 'email.identification', None), - ('Email de notification', 'email.notification', None), - ('Dernière mise à jour', 'last_update', None), - ('Dernière connexion', 'last_login', None), - ('Langues', 'languages', CommaSpaceListType), - ("Entreprise - A crée l'entreprise ? (Oui [1] / Non [0])", 'work.company.is_founder', BoolType), - ("Entreprise - A repris l'entreprise ? (Oui [1] / Non [0])", 'work.company.is_taker', BoolType), - ('Statut cotisant? (Oui [1] / Non [0])', 'is_contributing', BoolType), - ('Fidélité cotisant (nombre étoiles)', 'contribution_fidelity', None), - ('Délégué de promotion', 'is_delegate', BoolType), - ('Date de décès', 'death_date', None), - ('Adresse secondaire - Ligne 1', 'personal.address2.line_1', None), - ('Adresse secondaire - Ligne 2', 'personal.address2.line_2', None), - ('Adresse secondaire - Ligne 3', 'personal.address2.line_3', None), - ('Adresse secondaire - Ligne 4', 'personal.address2.line_4', None), - ('Adresse secondaire - Région', 'personal.address2.region', None), - ('Adresse secondaire - Ville', 'personal.address2.contry', None), - ('Adresse secondaire - État', 'personal.address2.state', None), - ('Adresse secondaire - Code postal', 'personal.address2.code', None), - ('Adresse secondaire - Cedex', 'personal.address2.cedex', None), - ('Adresse secondaire - Pays (ISO)', 'personal.address2.iso', None), - ('Téléphone fixe secondaire', 'personal.fix_phone_2', None), - ('Téléphone mobile secondaire', 'personal.cell_phone_2', None), - ('Compte X.org actif', 'xorg.is_active', YesNoBoolType), - ('Login X.org', 'xorg.login', None), - ('Matricule École', 'school.id', None), - ("Voie d'entrée", 'school.input', None), - ('Domaine du cursus', 'school.domain', None), - ('Intitulé du cursus', 'school.curriculum', None), - ('Corps actuel', 'corps.current', None), - ("Corps d'origine", 'corps.original', None), - ('Grade', 'corps.grade', None), - ('Surnom', 'nickname', None), - ('Seconde nationalité', 'nationality_2', None), - ('Troisième nationalité', 'nationality_3', None), - ('Mort pour la France', 'dead_for_france', None), - ("Section sportive à l'X", 'school.sport', None), - ('Ex-binets', 'school.binets', CommaListType), - ('Réception courrier', 'has_postal_mail', None), - ('Inscription aux newsletters', 'newsletters', CommaListType), - ('Référent', 'referrer.is', None), - ('Secteurs référent', 'referrer.sectors', None), - ('Commentaire référent', 'referrer.comment', None), - ('Info sup : commentaires divers', 'comments', None), - ('Url d\'appel à cotisation', 'url_contribution', None), + ("Identifiant (AlumnForce)", "id_af", None), + ("Identifiant (École)", "id_ax", None), + ("Prénom", "first_name", None), + ("Nom d'état-civil", "last_name", None), + ("Nom d'usage", "usage_name", None), + ("Civilité (Mme / Mlle / M.)", "civility", None), + ("Date de naissance", "birth_date", None), + ("Date de création de compte", "account_creation_date", None), + ("Date d'activation de compte", "account_activation_date", None), + ("Date de validation des CGU", "cgu_validation_date", None), + ("Adresse personnelle - Ligne 1", "personal.address.line_1", None), + ("Adresse personnelle - Ligne 2", "personal.address.line_2", None), + ("Adresse personnelle - Ligne 3", "personal.address.line_3", None), + ("Adresse personnelle - Ligne 4", "personal.address.line_4", None), + ("Adresse personnelle - Région", "personal.address.region", None), + ("Adresse personnelle - Ville", "personal.address.city", None), + ("Adresse personnelle - État", "personal.address.state", None), + ("Adresse personnelle - Code postal", "personal.address.code", None), + ("Adresse personnelle - Cedex", "personal.address.cedex", None), + ("Adresse personnelle - Pays (ISO)", "personal.address.country", None), + ("Adresse personnelle - NPAI (Oui [1] / Non [0])", "personal.address.bounced", BoolType), + ("Téléphone fixe personnel", "personal.fix_phone", None), + ("Téléphone mobile personnel", "personal.cell_phone", None), + ("Email personnel 1", "email.personal_1", None), + ("Email personnel 2", "email.personal_2", None), + ("Nationalité", "nationality", None), + ("Situation matrimoniale", "marital_status", None), + ("Membre décédé (Oui [1] / Non [0])", "is_dead", BoolType), + ( + "Statut (Non enregistré [0] / En cours d'activation [1] / Compte bloqué [2] / Compte activé [3] / Clé d'activation envoyée [4] / En attente de validation [5] / Ne souhaite pas activer son compte [6] / Compte expiré [8])", # noqa: E501 + "account_status", + None, + ), # noqa + ("Compte activé (Oui [1] / Non [0])", "is_activated", BoolType), + ( + "Type utilisateur (Diplômé(e) [1] / Personnel de l'association [3] / Élève / étudiant(e) [5] / Visiteur [7] / Membre associé [9] / Veuves/Veufs [10])", # noqa: E501 + "user_kind", + None, + ), # noqa + ( + "Rôle supplémentaire (Visiteur [1] / Administrateur total [3] / Diplômé [4] / Cotisant [5] / Élève et étudiant [7] / Abonné [17] / Membre associé [19] / Administrateur contenu [21] / Administrateur comptable [22] / Veuves/Veufs [26])", # noqa: E501 + "roles", + CommaListType, + ), # noqa + ("Forcer le statut de cotisant (Oui [1] / Non [0])", "force_contributor", BoolType), + ("Référence du diplôme", "school.degree_ref", None), + ("Ecole", "school.name", None), + ("Filière (L/M/D)", "school.stream", None), + ("Spécialisation", "speciality", None), + ("Parcours", "curriculum", None), + ("Niveau d'étude / Nombre d'années post bac", "study_level", None), + ("Mode", "mode", None), + ("A obtenu son diplôme ? (Oui [1] / Non [0])", "school.has_graduated", BoolType), + ("Date de promotion", "school.graduation_date", None), + ("Date d'intégration", "school.entry_date", None), + ("Publier données académiques", "school.publish_academic", BoolType), + ("Situation actuelle", "work.current_situation", None), + ("Poste actuel", "work.current_job", None), + ("Code Fontion / Métier", "work.job_code", None), + ("Fonction / Métier", "work.function", None), + ("Niveau du poste", "work.job_level", None), + ("Entreprise - Nom", "work.company.name", None), + ("Entreprise - Secteur d'activité", "work.company.sector", None), + ("Entreprise - Code SIRET", "work.company.siret", None), + ("Entreprise - Site internet", "work.company.website", None), + ("Adresse professionnelle - Ligne 1", "work.company.address.line_1", None), + ("Adresse professionnelle - Ligne 2", "work.company.address.line_2", None), + ("Adresse professionnelle - Ligne 3", "work.company.address.line_3", None), + ("Adresse professionnelle - Ligne 4", "work.company.address.line_4", None), + ("Adresse professionnelle - Région", "work.company.address.region", None), + ("Adresse professionnelle - Ville", "work.company.address.city", None), + ("Adresse professionnelle - État", "work.company.address.country", None), + ("Adresse professionnelle - Code postal", "work.company.address.code", None), + ("Adresse professionnelle - Cedex", "work.company.address.cedex", None), + ("Adresse professionnelle - Pays (ISO)", "work.company.address.iso", None), + ("Téléphone fixe professionnel", "work.company.fix_phone", None), + ("Téléphone mobile professionnel", "work.company.cell_phone", None), + ("Fax professionnel", "work.company.fax", None), + ("Email professionnel", "email.professional", None), + ("Début de l'expérience", "work.company.begin", None), + ("Fin de l'expérience", "work.company.end", None), + ("Type de contrat", "work.company.contract_kind", None), + ("Salaire réel", "work.company.wage", None), + ("Tranche de salaire", "work.company.wage_bracket", None), + ( + "Je souhaite contribuer à la vie de l'Association: CA, Colloque, PDX, Bal de l'X, Grand Magnan ... (Oui [1] / Non [0])", # noqa: E501 + "wish.ax", + BoolType, + ), # noqa + ( + "Je suis prêt à aider des camarades en transition professionnelle, ou porteurs de projet (Oui [1] / Non [0])", + "wish.help", + BoolType, + ), # noqa + ( + "Je suis intéressé par les programmes de parrainage et de mentoring (Oui [1] / Non [0])", + "wish.mentoring", + BoolType, + ), # noqa + ( + "Je suis prêt à donner un peu de temps pour aider des camarades, ou familles de camarades, en difficulté via la Caisse de Solidarité (Oui [1] / Non [0])", # noqa: E501 + "wish.solidarity", + BoolType, + ), # noqa + ("J'accepte de recevoir les mails de l'Association (Oui [1] / Non [0])", "wish.ax_emails", BoolType), + ( + "Je peux intervenir lors d'événements, pour animer un atelier, participer à une table ronde ou donner une conférence (Oui [1] / Non [0])", # noqa: E501 + "wish.animate", + BoolType, + ), # noqa + ("J'accepte de figurer dans l'annuaire papier (Oui [1] / Non [0])", "wish.in_directory", BoolType), + ("Je souhaite recevoir l'annuaire papier (Oui [1] / Non [0])", "wish.receive_directory", BoolType), + ("Email d'identification", "email.identification", None), + ("Email de notification", "email.notification", None), + ("Dernière mise à jour", "last_update", None), + ("Dernière connexion", "last_login", None), + ("Langues", "languages", CommaSpaceListType), + ("Entreprise - A crée l'entreprise ? (Oui [1] / Non [0])", "work.company.is_founder", BoolType), + ("Entreprise - A repris l'entreprise ? (Oui [1] / Non [0])", "work.company.is_taker", BoolType), + ("Statut cotisant? (Oui [1] / Non [0])", "is_contributing", BoolType), + ("Fidélité cotisant (nombre étoiles)", "contribution_fidelity", None), + ("Délégué de promotion", "is_delegate", BoolType), + ("Date de décès", "death_date", None), + ("Adresse secondaire - Ligne 1", "personal.address2.line_1", None), + ("Adresse secondaire - Ligne 2", "personal.address2.line_2", None), + ("Adresse secondaire - Ligne 3", "personal.address2.line_3", None), + ("Adresse secondaire - Ligne 4", "personal.address2.line_4", None), + ("Adresse secondaire - Région", "personal.address2.region", None), + ("Adresse secondaire - Ville", "personal.address2.contry", None), + ("Adresse secondaire - État", "personal.address2.state", None), + ("Adresse secondaire - Code postal", "personal.address2.code", None), + ("Adresse secondaire - Cedex", "personal.address2.cedex", None), + ("Adresse secondaire - Pays (ISO)", "personal.address2.iso", None), + ("Téléphone fixe secondaire", "personal.fix_phone_2", None), + ("Téléphone mobile secondaire", "personal.cell_phone_2", None), + ("Compte X.org actif", "xorg.is_active", YesNoBoolType), + ("Login X.org", "xorg.login", None), + ("Matricule École", "school.id", None), + ("Voie d'entrée", "school.input", None), + ("Domaine du cursus", "school.domain", None), + ("Intitulé du cursus", "school.curriculum", None), + ("Corps actuel", "corps.current", None), + ("Corps d'origine", "corps.original", None), + ("Grade", "corps.grade", None), + ("Surnom", "nickname", None), + ("Seconde nationalité", "nationality_2", None), + ("Troisième nationalité", "nationality_3", None), + ("Mort pour la France", "dead_for_france", None), + ("Section sportive à l'X", "school.sport", None), + ("Ex-binets", "school.binets", CommaListType), + ("Réception courrier", "has_postal_mail", None), + ("Inscription aux newsletters", "newsletters", CommaListType), + ("Référent", "referrer.is", None), + ("Secteurs référent", "referrer.sectors", None), + ("Commentaire référent", "referrer.comment", None), + ("Info sup : commentaires divers", "comments", None), + ("Url d'appel à cotisation", "url_contribution", None), ) diff --git a/xorgdata/alumnforce/management/commands/afsync.py b/xorgdata/alumnforce/management/commands/afsync.py index 547219c..e2b3ef2 100644 --- a/xorgdata/alumnforce/management/commands/afsync.py +++ b/xorgdata/alumnforce/management/commands/afsync.py @@ -2,8 +2,8 @@ import collections import datetime import ftplib -from pathlib import Path import re +from pathlib import Path from django.conf import settings from django.core.management import call_command @@ -18,7 +18,7 @@ def get_last_update_by_kind(): """ last_update_dates = collections.OrderedDict() for kind, _kind_name in models.ImportLog.KNOWN_EXPORT_KINDS: - qs = models.ImportLog.objects.filter(export_kind=kind).order_by('-date', '-is_incremental') + qs = models.ImportLog.objects.filter(export_kind=kind).order_by("-date", "-is_incremental") try: last_obj = qs[:1].get() except models.ImportLog.DoesNotExist: @@ -30,6 +30,7 @@ def get_last_update_by_kind(): class FtpConnection: """FTP connection to AlumnForce's FTP server""" + def __init__(self): # Connect to the FTPS server self.ftps = ftplib.FTP_TLS(settings.ALUMNFORCE_FTP_HOST) @@ -44,7 +45,9 @@ def __init__(self): def _dir_callback(self, line): """Callback for a dir command of a FTP client""" - matches = re.match(r'.* (export([a-z]+)-afbo-Polytechnique-X-([0-9]{4}[0-9]{2}[0-9]{2})\.csv(\.error)?)$', line) + matches = re.match( + r".* (export([a-z]+)-afbo-Polytechnique-X-([0-9]{4}[0-9]{2}[0-9]{2})\.csv(\.error)?)$", line + ) if not matches: # Ignore unknown files return @@ -61,25 +64,29 @@ def _dir_callback(self, line): def download_file(self, filename, out_path): """Download a file to the given output path""" - with open(out_path, 'wb') as fout: - self.ftps.retrbinary('RETR ' + filename, fout.write) + with open(out_path, "wb") as fout: + self.ftps.retrbinary("RETR " + filename, fout.write) class Command(BaseCommand): help = "Synchronise with AlumnForce's FTP server" def add_arguments(self, parser): - parser.add_argument('-n', '--dryrun', action='store_true', - help="show the files that would be applied, without updating anything") - parser.add_argument('--verbose', action='store_true', - help="show what is done") - parser.add_argument('--push-export', action='store_true', - help="export and and push it to Polytechnique.org's consumers") + parser.add_argument( + "-n", + "--dryrun", + action="store_true", + help="show the files that would be applied, without updating anything", + ) + parser.add_argument("--verbose", action="store_true", help="show what is done") + parser.add_argument( + "--push-export", action="store_true", help="export and and push it to Polytechnique.org's consumers" + ) def handle(self, *args, **options): last_update_dates = get_last_update_by_kind() - is_dryrun = options['dryrun'] + is_dryrun = options["dryrun"] if is_dryrun: self.stdout.write("Dry-run mode, nothing will be committed") @@ -90,7 +97,7 @@ def handle(self, *args, **options): # Connect to the FTPS server conn = FtpConnection() - if options['verbose']: + if options["verbose"]: self.stdout.write(self.style.SUCCESS("Connected to ftps://{}".format(settings.ALUMNFORCE_FTP_HOST))) download_dir_path = Path(settings.ALUMNFORCE_FTP_LOCAL_DIRECTORY) @@ -102,7 +109,7 @@ def handle(self, *args, **options): last_update_date = None last_was_incremental = None else: - last_update_date = lastup_data[0].strftime('%Y%m%d') + last_update_date = lastup_data[0].strftime("%Y%m%d") last_was_incremental = lastup_data[1] # Apply all possible files, sorting them by date @@ -119,7 +126,7 @@ def handle(self, *args, **options): file_date = datetime.date(year=int(date_str[:4]), month=int(date_str[4:6]), day=int(date_str[6:])) - if options['verbose']: + if options["verbose"]: self.stdout.write(self.style.SUCCESS("Downloading {}".format(filename))) dl_filepath = download_dir_path / filename @@ -146,16 +153,16 @@ def handle(self, *args, **options): self.stdout.write(self.style.SUCCESS("(not) applying file {}".format(repr(filename)))) continue - if not re.match(r'^[-0-9A-Za-z]+\.csv$', filename): + if not re.match(r"^[-0-9A-Za-z]+\.csv$", filename): raise CommandError("Unexpected bad filename {}".format(repr(filename))) - if options['verbose']: + if options["verbose"]: self.stdout.write(self.style.SUCCESS("Applying {}".format(dl_filepath))) - call_command('importcsv', dl_filepath, kind=kind) + call_command("importcsv", dl_filepath, kind=kind) else: - call_command('importcsv', dl_filepath, kind=kind, verbosity=0) + call_command("importcsv", dl_filepath, kind=kind, verbosity=0) - #dl_filepath.unlink() # we keep an archive of all files downloaded + # dl_filepath.unlink() # we keep an archive of all files downloaded - if options['push_export']: - call_command('exportforauth', push=True) + if options["push_export"]: + call_command("exportforauth", push=True) diff --git a/xorgdata/alumnforce/management/commands/exportforauth.py b/xorgdata/alumnforce/management/commands/exportforauth.py index 0192e6c..2a16342 100644 --- a/xorgdata/alumnforce/management/commands/exportforauth.py +++ b/xorgdata/alumnforce/management/commands/exportforauth.py @@ -13,41 +13,39 @@ class Command(BaseCommand): help = "Export data which is used by X.org authentication project" def add_arguments(self, parser): - parser.add_argument('--push', action='store_true', - help="push the exported data to {}".format(settings.XORGAUTH_HOST)) + parser.add_argument( + "--push", action="store_true", help="push the exported data to {}".format(settings.XORGAUTH_HOST) + ) def handle(self, *args, **options): # Avoid duplicated X.org login in the exported data duplicated_xorg_id = ( - models.Account.objects - .filter(deleted_since=None) + models.Account.objects.filter(deleted_since=None) .exclude(xorg_id=None) - .values('xorg_id') - .annotate(count=Count('af_id')) - .values('xorg_id') + .values("xorg_id") + .annotate(count=Count("af_id")) + .values("xorg_id") .filter(count__gt=1) ) # Export all accounts that have not been deleted and that have a X.org login - accounts_qs = ( - models.Account.objects - .filter(deleted_since=None) - .exclude(xorg_id=None, xorg_id__in=duplicated_xorg_id) + accounts_qs = models.Account.objects.filter(deleted_since=None).exclude( + xorg_id=None, xorg_id__in=duplicated_xorg_id ) def export_account(account): roles = account.get_additional_roles() return { - 'xorg_id': account.xorg_id, - 'af_id': account.af_id, - 'ax_contributor': models.Account.ROLE_CONTRIBUTOR in roles, - 'axjr_subscribed': models.Account.ROLE_SUBSCRIBED in roles, - 'last_updated': account.last_update.strftime('%Y-%m-%d') + "xorg_id": account.xorg_id, + "af_id": account.af_id, + "ax_contributor": models.Account.ROLE_CONTRIBUTOR in roles, + "axjr_subscribed": models.Account.ROLE_SUBSCRIBED in roles, + "last_updated": account.last_update.strftime("%Y-%m-%d"), } exported_data = [export_account(account) for account in accounts_qs] - if not options['push']: + if not options["push"]: # Show the exported data, without exporting it self.stdout.write(json.dumps(exported_data)) return @@ -59,13 +57,15 @@ def export_account(account): page_size = 2000 for page_offset in range(0, len(exported_data), page_size): req = urllib.request.Request( - 'https://{}/sync/axdata'.format(settings.XORGAUTH_HOST), - data=json.dumps({ - 'secret': settings.XORGAUTH_PASSWORD, - 'data': exported_data[page_offset:page_offset + page_size], - }).encode('ascii'), + "https://{}/sync/axdata".format(settings.XORGAUTH_HOST), + data=json.dumps( + { + "secret": settings.XORGAUTH_PASSWORD, + "data": exported_data[page_offset : page_offset + page_size], + } + ).encode("ascii"), headers={ - 'Content-type': 'application/json', + "Content-type": "application/json", }, ) opener = urllib.request.build_opener() diff --git a/xorgdata/alumnforce/management/commands/importallusers.py b/xorgdata/alumnforce/management/commands/importallusers.py index d7c375e..3e352b0 100644 --- a/xorgdata/alumnforce/management/commands/importallusers.py +++ b/xorgdata/alumnforce/management/commands/importallusers.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- """Import data from a full export from AlumnForce website""" + import datetime import os.path import re @@ -18,7 +19,7 @@ def get_export_date_from_filename(file_path): Example of path: export-users-20190204-133700.csv """ file_name = os.path.basename(file_path) - match = re.match(r'.*-([0-9]{4})([0-9]{2})([0-9]{2})-[0-9]{6}\.csv$', file_name) + match = re.match(r".*-([0-9]{4})([0-9]{2})([0-9]{2})-[0-9]{6}\.csv$", file_name) if match: year, month, day = match.groups() return datetime.date(year=int(year), month=int(month), day=int(day)) @@ -29,21 +30,21 @@ class Command(BaseCommand): help = "Import data from a full export of users from AlumnForce database" def add_arguments(self, parser): - parser.add_argument('csvfile', type=str, - help="path to CSV file to load") - parser.add_argument('--date', type=str, - help="date associate with the export (by default: use the file name)") + parser.add_argument("csvfile", type=str, help="path to CSV file to load") + parser.add_argument("--date", type=str, help="date associate with the export (by default: use the file name)") def handle(self, *args, **options): - file_path = options['csvfile'] - if options['date']: - file_date = datetime.datetime.strptime(options['date'], '%Y-%m-%d').date() + file_path = options["csvfile"] + if options["date"]: + file_date = datetime.datetime.strptime(options["date"], "%Y-%m-%d").date() # Compare with a potential file date maybe_file_date = get_export_date_from_filename(file_path) if maybe_file_date and maybe_file_date != file_date: - self.stdout.write(self.style.WARNING( - "Forcing date %s that mismatches with file date %s" % - (file_date, maybe_file_date))) + self.stdout.write( + self.style.WARNING( + "Forcing date %s that mismatches with file date %s" % (file_date, maybe_file_date) + ) + ) else: file_date = get_export_date_from_filename(file_path) if not file_date: @@ -56,66 +57,66 @@ def handle(self, *args, **options): json_obj = AlumnForceDataC2J.import_csv_file(file_path, keep_empty=True) for user_data in json_obj.content: # Prepare a dict for insertion into the Django database - af_id = int(user_data['id_af']) + af_id = int(user_data["id_af"]) fields = { - 'ax_id': user_data['id_ax'] or None, - 'first_name': user_data['first_name'], - 'last_name': user_data['last_name'], - 'common_name': user_data['usage_name'], - 'civility': user_data['civility'], - 'birthdate': parse_french_date(user_data['birth_date']), - 'address_1': user_data['personal']['address']['line_1'], - 'address_2': user_data['personal']['address']['line_2'], - 'address_3': user_data['personal']['address']['line_3'], - 'address_4': user_data['personal']['address']['line_4'], - 'address_postcode': user_data['personal']['address']['code'], - 'address_city': user_data['personal']['address']['city'], - 'address_state': user_data['personal']['address']['state'], - 'address_country': user_data['personal']['address']['country'], - 'address_npai': user_data['personal']['address']['bounced'], - 'phone_personnal': user_data['personal']['fix_phone'], - 'phone_mobile': user_data['personal']['cell_phone'], - 'email_1': user_data['email']['personal_1'], - 'email_2': user_data['email']['personal_2'], - 'nationality': user_data['nationality'], - 'nationality_2': user_data['nationality_2'], - 'nationality_3': user_data['nationality_3'], - 'dead': user_data['is_dead'], - 'deathdate': parse_french_date(user_data['death_date']), - 'dead_for_france': user_data['dead_for_france'], - 'user_kind': user_data['user_kind'], - 'additional_roles': '', - 'xorg_id': user_data['xorg']['login'] or None, - 'school_id': user_data['school']['id'], - 'admission_path': user_data['school']['input'], - 'cursus_domain': user_data['school']['domain'], - 'cursus_name': user_data['school']['name'], - 'corps_current': user_data['corps']['current'], - 'corps_origin': user_data['corps']['original'], - 'corps_grade': user_data['corps']['grade'], - 'nickname': user_data['nickname'], - 'sport_section': user_data['school']['sport'], - 'binets': ','.join(user_data['school']['binets'] or []), - 'mail_reception': user_data['has_postal_mail'], - 'newsletter_inscriptions': ','.join(user_data['newsletters'] or []), - 'last_update': file_date, - 'deleted_since': None, + "ax_id": user_data["id_ax"] or None, + "first_name": user_data["first_name"], + "last_name": user_data["last_name"], + "common_name": user_data["usage_name"], + "civility": user_data["civility"], + "birthdate": parse_french_date(user_data["birth_date"]), + "address_1": user_data["personal"]["address"]["line_1"], + "address_2": user_data["personal"]["address"]["line_2"], + "address_3": user_data["personal"]["address"]["line_3"], + "address_4": user_data["personal"]["address"]["line_4"], + "address_postcode": user_data["personal"]["address"]["code"], + "address_city": user_data["personal"]["address"]["city"], + "address_state": user_data["personal"]["address"]["state"], + "address_country": user_data["personal"]["address"]["country"], + "address_npai": user_data["personal"]["address"]["bounced"], + "phone_personnal": user_data["personal"]["fix_phone"], + "phone_mobile": user_data["personal"]["cell_phone"], + "email_1": user_data["email"]["personal_1"], + "email_2": user_data["email"]["personal_2"], + "nationality": user_data["nationality"], + "nationality_2": user_data["nationality_2"], + "nationality_3": user_data["nationality_3"], + "dead": user_data["is_dead"], + "deathdate": parse_french_date(user_data["death_date"]), + "dead_for_france": user_data["dead_for_france"], + "user_kind": user_data["user_kind"], + "additional_roles": "", + "xorg_id": user_data["xorg"]["login"] or None, + "school_id": user_data["school"]["id"], + "admission_path": user_data["school"]["input"], + "cursus_domain": user_data["school"]["domain"], + "cursus_name": user_data["school"]["name"], + "corps_current": user_data["corps"]["current"], + "corps_origin": user_data["corps"]["original"], + "corps_grade": user_data["corps"]["grade"], + "nickname": user_data["nickname"], + "sport_section": user_data["school"]["sport"], + "binets": ",".join(user_data["school"]["binets"] or []), + "mail_reception": user_data["has_postal_mail"], + "newsletter_inscriptions": ",".join(user_data["newsletters"] or []), + "last_update": file_date, + "deleted_since": None, } - if fields['civility'] == 'M.': + if fields["civility"] == "M.": # Normalize civility, in order to share the same format as incremental exports - fields['civility'] = 'M' - if fields['school_id'] == '0': + fields["civility"] = "M" + if fields["school_id"] == "0": # Normalize school ID - fields['school_id'] = '' + fields["school_id"] = "" - for key in ('nationality', 'nationality_2', 'nationality_3'): + for key in ("nationality", "nationality_2", "nationality_3"): # Make an unfilled field blank - if fields[key] == 'Non renseigné': - fields[key] = '' + if fields[key] == "Non renseigné": + fields[key] = "" - if user_data['roles']: + if user_data["roles"]: # Format the additional roles as a list of integers - fields['additional_roles'] = ','.join(user_data['roles']) + fields["additional_roles"] = ",".join(user_data["roles"]) models.Account.objects.update_or_create(af_id=af_id, defaults=fields) if af_id in deleted_account_ids: deleted_account_ids.remove(af_id) @@ -132,7 +133,7 @@ def handle(self, *args, **options): self.stdout.write(self.style.SUCCESS(message)) models.ImportLog.objects.create( date=file_date, - export_kind='users', + export_kind="users", is_incremental=False, error=models.ImportLog.SUCCESS, num_modified=num_users, diff --git a/xorgdata/alumnforce/management/commands/importcsv.py b/xorgdata/alumnforce/management/commands/importcsv.py index 8443f85..4bd2006 100644 --- a/xorgdata/alumnforce/management/commands/importcsv.py +++ b/xorgdata/alumnforce/management/commands/importcsv.py @@ -3,30 +3,30 @@ import csv import datetime +import hashlib import os.path import re -import hashlib from django.conf import settings -from django.core.management.base import BaseCommand, CommandError from django.core.mail import send_mail +from django.core.management.base import BaseCommand, CommandError from xorgdata.alumnforce import models def bool_or_none(txt): - if txt == '': + if txt == "": return None - elif txt in ('0', '1'): + elif txt in ("0", "1"): return bool(int(txt)) else: raise ValueError("invalid bool value {}".format(repr(txt))) def int_or_none(txt): - if txt == '': + if txt == "": return None - elif re.match(r'^[0-9]+$', txt): + elif re.match(r"^[0-9]+$", txt): return int(txt) else: raise ValueError("invalid integer value: {}".format(repr(txt))) @@ -37,15 +37,15 @@ def phone_indicator(txt): It may start with +, which is why int_or_none is not used. """ - if txt == '': + if txt == "": return None - elif re.match(r'^\+?[0-9]+$', txt): + elif re.match(r"^\+?[0-9]+$", txt): return int(txt) else: raise ValueError("invalid phone indicator value: {}".format(repr(txt))) -FRENCH_DATE_RE = re.compile(r'(?P\d{1,2})/(?P\d{1,2})/(?P\d{4})$') +FRENCH_DATE_RE = re.compile(r"(?P\d{1,2})/(?P\d{1,2})/(?P\d{4})$") def parse_french_date(value): @@ -61,105 +61,105 @@ def parse_french_date(value): # Mapping from CSV columns to database fields ALUMNFORCE_USER_FIELDS = { - 'Identifiant AF': ('af_id', int), - 'Identifiant école': ('ax_id', str), - 'Prénom': ('first_name', str), - 'Nom d\'état civil': ('last_name', str), - 'Nom d\'usage': ('common_name', str), - 'Civilité': ('civility', str), - 'Date de naissance': ('birthdate', parse_french_date), - 'Adresse personnelle - Ligne 1': ('address_1', str), - 'Adresse personnelle - Ligne 2': ('address_2', str), - 'Adresse personnelle - Ligne 3': ('address_3', str), - 'Adresse personnelle - Ligne 4': ('address_4', str), - 'Adresse personnelle - Code Postal': ('address_postcode', str), - 'Adresse personnelle - Ville': ('address_city', str), - 'Adresse personnelle - État': ('address_state', str), - 'Adresse personnelle - Pays': ('address_country', str), - 'NPAI': ('address_npai', bool_or_none), - 'Téléphone fixe personnel': ('phone_personnal', str), - 'Téléphone mobile personnel': ('phone_mobile', str), - 'Email personnel 1': ('email_1', str), - 'Email personnel 2': ('email_2', str), - 'Nationalité': ('nationality', str), - 'Décédé': ('dead', bool_or_none), - 'Date de décès': ('deathdate', parse_french_date), - 'Type d\'utilisateur': ('user_kind', int), - 'Rôles supplémentaires': ('additional_roles', str), # comma-separated integers - 'Login X.org': ('xorg_id', str), - 'Matricule école': ('school_id', str), - 'Voie d\'entrée': ('admission_path', str), - 'Domaine du cursus': ('cursus_domain', str), - 'Intitulé du cursus': ('cursus_name', str), - 'Corps actuel': ('corps_current', str), - 'Corps d\'origine': ('corps_origin', str), - 'Grade': ('corps_grade', str), - 'Surnom': ('nickname', str), - 'Seconde nationalité': ('nationality_2', str), - 'Troisième nationalité': ('nationality_3', str), - 'Mort pour la france': ('dead_for_france', str), - 'Sections sportive à l’X': ('sport_section', str), - 'Ex-binets': ('binets', str), - 'Réception courrier': ('mail_reception', str), - 'Inscription aux newsletters': ('newsletter_inscriptions', str), - 'URL de la photo de profil': ('profile_picture_url', str), + "Identifiant AF": ("af_id", int), + "Identifiant école": ("ax_id", str), + "Prénom": ("first_name", str), + "Nom d'état civil": ("last_name", str), + "Nom d'usage": ("common_name", str), + "Civilité": ("civility", str), + "Date de naissance": ("birthdate", parse_french_date), + "Adresse personnelle - Ligne 1": ("address_1", str), + "Adresse personnelle - Ligne 2": ("address_2", str), + "Adresse personnelle - Ligne 3": ("address_3", str), + "Adresse personnelle - Ligne 4": ("address_4", str), + "Adresse personnelle - Code Postal": ("address_postcode", str), + "Adresse personnelle - Ville": ("address_city", str), + "Adresse personnelle - État": ("address_state", str), + "Adresse personnelle - Pays": ("address_country", str), + "NPAI": ("address_npai", bool_or_none), + "Téléphone fixe personnel": ("phone_personnal", str), + "Téléphone mobile personnel": ("phone_mobile", str), + "Email personnel 1": ("email_1", str), + "Email personnel 2": ("email_2", str), + "Nationalité": ("nationality", str), + "Décédé": ("dead", bool_or_none), + "Date de décès": ("deathdate", parse_french_date), + "Type d'utilisateur": ("user_kind", int), + "Rôles supplémentaires": ("additional_roles", str), # comma-separated integers + "Login X.org": ("xorg_id", str), + "Matricule école": ("school_id", str), + "Voie d'entrée": ("admission_path", str), + "Domaine du cursus": ("cursus_domain", str), + "Intitulé du cursus": ("cursus_name", str), + "Corps actuel": ("corps_current", str), + "Corps d'origine": ("corps_origin", str), + "Grade": ("corps_grade", str), + "Surnom": ("nickname", str), + "Seconde nationalité": ("nationality_2", str), + "Troisième nationalité": ("nationality_3", str), + "Mort pour la france": ("dead_for_france", str), + "Sections sportive à l’X": ("sport_section", str), + "Ex-binets": ("binets", str), + "Réception courrier": ("mail_reception", str), + "Inscription aux newsletters": ("newsletter_inscriptions", str), + "URL de la photo de profil": ("profile_picture_url", str), } ALUMNFORCE_USERDEGREE_FIELDS = { - 'Identifiant AF': ('af_id', int), - 'Identifiant école': ('ax_id', str), - 'Référence du diplôme': ('diploma_reference', str), - 'A obtenu son diplôme ?': ('diplomed', bool_or_none), - 'Date d\'obtention du diplôme': ('diplomation_date', parse_french_date), - 'Mode de formation': ('domain', str), - 'Cycle': ('name', str), + "Identifiant AF": ("af_id", int), + "Identifiant école": ("ax_id", str), + "Référence du diplôme": ("diploma_reference", str), + "A obtenu son diplôme ?": ("diplomed", bool_or_none), + "Date d'obtention du diplôme": ("diplomation_date", parse_french_date), + "Mode de formation": ("domain", str), + "Cycle": ("name", str), } ALUMNFORCE_USERJOB_FIELDS = { - 'Identifiant AF': ('af_id', int), - 'Identifiant école': ('ax_id', str), - 'Titre du poste': ('title', str), - 'Fonction dans l\'entreprise': ('role', str), - 'Nom de l\'entreprise': ('company_name', str), - 'Adresse professionnelle - Ligne 1': ('address_1', str), - 'Adresse professionnelle - Ligne 2': ('address_2', str), - 'Adresse professionnelle - Ligne 3': ('address_3', str), - 'Adresse professionnelle - Ligne 4': ('address_4', str), - 'Adresse professionnelle - Code postal': ('address_postcode', str), - 'Adresse professionnelle - Ville': ('address_city', str), - 'Adresse professionnelle - Pays': ('address_country', str), - 'Indicateur téléphone fixe professionnel': ('phone_indicator', phone_indicator), - 'Téléphone fixe professionnel': ('phone_number', str), - 'Indicateur téléphone mobile professionnel': ('mobile_phone_indicator', phone_indicator), - 'Téléphone mobile professionnel': ('mobile_phone_number', str), - 'Fax professionnel': ('fax', str), - 'Email professionnel': ('email', str), - 'Date de début de l\'expérience': ('start_date', parse_french_date), - 'Date de fin de l\'expérience': ('end_date', parse_french_date), - 'Type de contrat': ('contract_kind', str), - 'Poste actuel ?': ('current', bool_or_none), - 'J\'ai créé cette entreprise ?': ('creator_of_company', bool_or_none), - 'J\'ai repris cette entreprise ?': ('buyer_of_company', bool_or_none), + "Identifiant AF": ("af_id", int), + "Identifiant école": ("ax_id", str), + "Titre du poste": ("title", str), + "Fonction dans l'entreprise": ("role", str), + "Nom de l'entreprise": ("company_name", str), + "Adresse professionnelle - Ligne 1": ("address_1", str), + "Adresse professionnelle - Ligne 2": ("address_2", str), + "Adresse professionnelle - Ligne 3": ("address_3", str), + "Adresse professionnelle - Ligne 4": ("address_4", str), + "Adresse professionnelle - Code postal": ("address_postcode", str), + "Adresse professionnelle - Ville": ("address_city", str), + "Adresse professionnelle - Pays": ("address_country", str), + "Indicateur téléphone fixe professionnel": ("phone_indicator", phone_indicator), + "Téléphone fixe professionnel": ("phone_number", str), + "Indicateur téléphone mobile professionnel": ("mobile_phone_indicator", phone_indicator), + "Téléphone mobile professionnel": ("mobile_phone_number", str), + "Fax professionnel": ("fax", str), + "Email professionnel": ("email", str), + "Date de début de l'expérience": ("start_date", parse_french_date), + "Date de fin de l'expérience": ("end_date", parse_french_date), + "Type de contrat": ("contract_kind", str), + "Poste actuel ?": ("current", bool_or_none), + "J'ai créé cette entreprise ?": ("creator_of_company", bool_or_none), + "J'ai repris cette entreprise ?": ("buyer_of_company", bool_or_none), } ALUMNFORCE_GROUP_FIELDS = { - 'Identifiant AF': ('af_id', int), - 'Matricule AX': ('ax_id', str), - 'URL du groupe': ('url', str), - 'Nom du groupe': ('name', str), - 'Catégorie du groupe': ('category', str), + "Identifiant AF": ("af_id", int), + "Matricule AX": ("ax_id", str), + "URL du groupe": ("url", str), + "Nom du groupe": ("name", str), + "Catégorie du groupe": ("category", str), } ALUMNFORCE_GROUPMEMBER_FIELDS = { - 'Identifiant AF utilisateur': ('user_id', int), - 'Matricule AX': ('user_ax_id', str), - 'Identifiant AF groupe': ('group_id', int), - 'Rôle dans le groupe': ('role', str), + "Identifiant AF utilisateur": ("user_id", int), + "Matricule AX": ("user_ax_id", str), + "Identifiant AF groupe": ("group_id", int), + "Rôle dans le groupe": ("role", str), } ALUMNFORCE_GROUPMEMBER_ROLES = { - 'banni': 'banned', - 'désinscrit': 'unsubscribed', - 'invité': 'invited', - 'membre': 'member', - 'modérateur': 'moderator', - 'responsable': 'responsible', - 'sur liste': 'onlist', + "banni": "banned", + "désinscrit": "unsubscribed", + "invité": "invited", + "membre": "member", + "modérateur": "moderator", + "responsable": "responsible", + "sur liste": "onlist", } @@ -173,7 +173,7 @@ def get_export_kind_from_filename(file_path): Example of path: downloads/ftp/exportusers-afbo-Polytechnique-X-20190323.csv """ file_name = os.path.basename(file_path) - match = re.match(r'^export([a-z]+)-afbo[^.]*\.csv$', file_name) + match = re.match(r"^export([a-z]+)-afbo[^.]*\.csv$", file_name) if match: kind = match.group(1) if kind not in KNOWN_EXPORT_KINDS: @@ -188,7 +188,7 @@ def get_export_date_from_filename(file_path): Example of path: downloads/ftp/exportusers-afbo-Polytechnique-X-20190323.csv """ file_name = os.path.basename(file_path) - match = re.match(r'.*([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9])\.csv$', file_name) + match = re.match(r".*([0-9][0-9][0-9][0-9])([0-9][0-9])([0-9][0-9])\.csv$", file_name) if match: year, month, day = match.groups() return datetime.date(year=int(year), month=int(month), day=int(day)) @@ -197,33 +197,17 @@ def get_export_date_from_filename(file_path): def compute_current_problem_file_path(kind, id): id_str = str(id) - directory = os.path.join( - settings.PERSISTENT_DIRECTORY, - "current_problems_by_id", - kind) + directory = os.path.join(settings.PERSISTENT_DIRECTORY, "current_problems_by_id", kind) os.makedirs(directory, exist_ok=True) - return os.path.join( - directory, - id_str + ".rej" - ) + return os.path.join(directory, id_str + ".rej") def compute_problem_archive_file_path(kind, id, csv_file_path, state, account_str, hash): id_str = str(id) - directory = os.path.join( - settings.PERSISTENT_DIRECTORY, - "problem_archive", - kind) + directory = os.path.join(settings.PERSISTENT_DIRECTORY, "problem_archive", kind) os.makedirs(directory, exist_ok=True) return os.path.join( - directory, - "__".join([ - id_str, - os.path.basename(csv_file_path), - state, - account_str, - hash - ]) + ".txt" + directory, "__".join([id_str, os.path.basename(csv_file_path), state, account_str, hash]) + ".txt" ) @@ -238,9 +222,11 @@ def compute_problem_archive_file_path(kind, id, csv_file_path, state, account_st # We want: # - For clarity, whatever the file content, have a log in https://data.m4x.org/admin/alumnforce/importlog/ . # - Malformed lines cause explicit log+mail, but do not prevent importing other good lines in the file. -# - For debugging, at any time the set of currently invalid lines (indexed by the first field AF_ID) is accessible (filesystem + mail) +# - For debugging, at any time the set of currently invalid lines (indexed by the first field AF_ID) is accessible +# (filesystem + mail) # - To keep sync on, malformed lines do not prevent moving on to the next file when available. -# - When a newer file provides a valid fixed line for any AF_ID, the now obsolete incident is removed from the set of invalid lines. +# - When a newer file provides a valid fixed line for any AF_ID, the now obsolete incident is removed from the set of +# invalid lines. # - An archive is kept of lines that caused an issue and line that fixed it. # To properly record the various cases, we need a hash and an id. @@ -257,11 +243,12 @@ def compute_problem_archive_file_path(kind, id, csv_file_path, state, account_st # So, we wait for the whole file to be processed and only then we figure out which users are affected. # For this reason, load_csv returns a tuple: a parse_report and the value. + def load_csv(kind, csv_file_path, fields): - with open(csv_file_path, 'r', encoding='utf-8') as line_stream: + with open(csv_file_path, "r", encoding="utf-8") as line_stream: all_lines = line_stream.readlines() - reader = csv.reader(all_lines, delimiter='\t', quoting=csv.QUOTE_NONE, escapechar='\\', strict=True) + reader = csv.reader(all_lines, delimiter="\t", quoting=csv.QUOTE_NONE, escapechar="\\", strict=True) header_row = [] conversions = [] for row in reader: @@ -273,8 +260,9 @@ def load_csv(kind, csv_file_path, fields): conversions.append(fields.get(col_name, col_name)[1]) # Sanity check - assert len(set(header_row)) == len(header_row), \ - "There are columns which are not unique in {}".format(csv_file_path) + assert len(set(header_row)) == len(header_row), "There are columns which are not unique in {}".format( + csv_file_path + ) continue # Reader.line_num is indeed what we need, not a record count, @@ -282,11 +270,10 @@ def load_csv(kind, csv_file_path, fields): # Also reader provides 1-based line number, other need zero-based, so subtract one. csv_raw_line_num = reader.line_num - 1 csv_raw_line = all_lines[csv_raw_line_num] - line_hash = hashlib.sha256(csv_raw_line.encode('utf-8')).hexdigest() + line_hash = hashlib.sha256(csv_raw_line.encode("utf-8")).hexdigest() - account_label_for_filename = "unknown" problems = [] - af_id = None, + af_id = (None,) parse_report = { "problems": problems, "af_id": af_id, @@ -295,12 +282,13 @@ def load_csv(kind, csv_file_path, fields): "line_num": csv_raw_line_num, "line_hash": line_hash, "line": csv_raw_line, - "line_tabs": csv_raw_line.replace('\t', ''), + "line_tabs": csv_raw_line.replace("\t", ""), } value = None try: - assert len(row) == len(header_row), \ + assert len(row) == len(header_row), ( f"Line has {len(row)} items but the header has {len(header_row)} items." + ) # convert the values as appropriate row = [conv(val) for (val, conv) in zip(row, conversions)] value = dict(zip(header_row, row)) @@ -314,22 +302,23 @@ def load_csv(kind, csv_file_path, fields): # Time to extract what we can from the line. try: - # The pattern is to update the parse_report record with the most relevant message in case the next step fails. + # The pattern is to update the parse_report record with the most relevant message in case the next + # step fails. # If we're interrupted at any point below, the error record will just # remain with the last information. failure = "cannot extract AF_ID" - if '\t' not in csv_raw_line: + if "\t" not in csv_raw_line: failure = "no tab character" raise Exception("No tab character") # Alumnforce IDs are currently 5 figures, using 9 leave some room. - af_id_str = csv_raw_line.split('\t')[0][0:9] + af_id_str = csv_raw_line.split("\t")[0][0:9] af_id = int(af_id_str) except Exception as exc2: problems.append(failure) problems.append(exc2) - parse_report['af_id'] = af_id + parse_report["af_id"] = af_id yield (parse_report, value) @@ -337,10 +326,8 @@ class Command(BaseCommand): help = "Import a CSV file with accounts data into the database" def add_arguments(self, parser): - parser.add_argument('-k', '--kind', type=str, choices=KNOWN_EXPORT_KINDS, - help="Kind of csv filed to load") - parser.add_argument('csvfile', nargs='+', type=str, - help="path to CSV file to load") + parser.add_argument("-k", "--kind", type=str, choices=KNOWN_EXPORT_KINDS, help="Kind of csv filed to load") + parser.add_argument("csvfile", nargs="+", type=str, help="path to CSV file to load") def log_success(self, file_date, file_kind, num_values, file_path, facts): """Log a successful import""" @@ -371,9 +358,9 @@ def log_warning(self, file_date, file_kind, message): ) def handle(self, *args, **options): - self.verbosity = options['verbosity'] + self.verbosity = options["verbosity"] - timestamp_start = datetime.datetime.utcnow() + timestamp_start = datetime.datetime.now(datetime.UTC) kinds_involved_in_imported_files = set() parse_reports_by_kind = {} @@ -382,7 +369,7 @@ def handle(self, *args, **options): problem_changes_all_files = {} resolved_to_be_also_in_report = [] - for file_path in options['csvfile']: + for file_path in options["csvfile"]: file_date = get_export_date_from_filename(file_path) if not file_date: raise CommandError("Unable to find a date in file path %r" % file_path) @@ -391,19 +378,16 @@ def handle(self, *args, **options): file_kind = get_export_kind_from_filename(file_path) except ValueError as exc: # Forward the exception if there is no default value - if not options['kind']: + if not options["kind"]: raise CommandError(str(exc)) file_kind = None if not file_kind: - if not options['kind']: - raise CommandError( - "Unable to find the kind of %r, use --kind option" % file_path) - file_kind = options['kind'] - elif options['kind'] and file_kind != options['kind']: - raise CommandError( - "Incompatible kind for file %r: %r != %r" % ( - file_path, file_kind, options['kind'])) + if not options["kind"]: + raise CommandError("Unable to find the kind of %r, use --kind option" % file_path) + file_kind = options["kind"] + elif options["kind"] and file_kind != options["kind"]: + raise CommandError("Incompatible kind for file %r: %r != %r" % (file_path, file_kind, options["kind"])) kinds_involved_in_imported_files.add(file_kind) parse_reports_this_kind = [] @@ -411,124 +395,127 @@ def handle(self, *args, **options): if file_kind == "users": num_values = 0 - for (parse_report, value) in load_csv(file_kind, file_path, ALUMNFORCE_USER_FIELDS): + for parse_report, value in load_csv(file_kind, file_path, ALUMNFORCE_USER_FIELDS): parse_reports_this_kind.append(parse_report) if not value: continue - value['last_update'] = file_date - value['deleted_since'] = None - for key in ('nationality', 'nationality_2', 'nationality_3'): + value["last_update"] = file_date + value["deleted_since"] = None + for key in ("nationality", "nationality_2", "nationality_3"): # Make an unfilled field blank - if value[key] == 'Non renseigné': - value[key] = '' - if value['school_id'] == '0': - value['school_id'] = '' - if value['xorg_id'] == '': - value['xorg_id'] = None - if value['profile_picture_url'].startswith('/'): - value['profile_picture_url'] = 'https://ax.polytechnique.org' + value['profile_picture_url'] - models.Account.objects.update_or_create(af_id=value['af_id'], defaults=value) + if value[key] == "Non renseigné": + value[key] = "" + if value["school_id"] == "0": + value["school_id"] = "" + if value["xorg_id"] == "": + value["xorg_id"] = None + if value["profile_picture_url"].startswith("/"): + value["profile_picture_url"] = "https://ax.polytechnique.org" + value["profile_picture_url"] + models.Account.objects.update_or_create(af_id=value["af_id"], defaults=value) num_values += 1 elif file_kind == "userdegrees": num_values = 0 seen_accounts = {} - for (parse_report, value) in load_csv(file_kind, file_path, ALUMNFORCE_USERDEGREE_FIELDS): + for parse_report, value in load_csv(file_kind, file_path, ALUMNFORCE_USERDEGREE_FIELDS): parse_reports_this_kind.append(parse_report) if not value: continue - account = seen_accounts.get(value['af_id']) + account = seen_accounts.get(value["af_id"]) if account is None: try: - account = models.Account.objects.get(af_id=value['af_id']) + account = models.Account.objects.get(af_id=value["af_id"]) except models.Account.DoesNotExist: self.log_warning( - file_date, file_kind, + file_date, + file_kind, "Unable to find user with AF ID {} (AX ID {})".format( - value['af_id'], repr(value['ax_id']) - )) + value["af_id"], repr(value["ax_id"]) + ), + ) continue - seen_accounts[value['af_id']] = account + seen_accounts[value["af_id"]] = account # Remove previous degrees when an account is seen for the first time account.degrees.all().delete() # Insert a degree - del value['af_id'] - del value['ax_id'] - value['last_update'] = file_date + del value["af_id"] + del value["ax_id"] + value["last_update"] = file_date account.degrees.create(**value) num_values += 1 elif file_kind == "userjobs": num_values = 0 seen_accounts = {} - for (parse_report, value) in load_csv(file_kind, file_path, ALUMNFORCE_USERJOB_FIELDS): + for parse_report, value in load_csv(file_kind, file_path, ALUMNFORCE_USERJOB_FIELDS): parse_reports_this_kind.append(parse_report) if not value: continue - account = seen_accounts.get(value['af_id']) + account = seen_accounts.get(value["af_id"]) if account is None: try: - account = models.Account.objects.get(af_id=value['af_id']) + account = models.Account.objects.get(af_id=value["af_id"]) except models.Account.DoesNotExist: self.log_warning( - file_date, file_kind, + file_date, + file_kind, "Unable to find user with AF ID {} (AX ID {})".format( - value['af_id'], repr(value['ax_id']) - )) + value["af_id"], repr(value["ax_id"]) + ), + ) continue - seen_accounts[value['af_id']] = account + seen_accounts[value["af_id"]] = account # Remove previous jobs when an account is seen for the first time account.jobs.all().delete() # Insert a job - del value['af_id'] - del value['ax_id'] - value['last_update'] = file_date + del value["af_id"] + del value["ax_id"] + value["last_update"] = file_date account.jobs.create(**value) num_values += 1 elif file_kind == "groups": num_values = 0 - for (parse_report, value) in load_csv(file_kind, file_path, ALUMNFORCE_GROUP_FIELDS): + for parse_report, value in load_csv(file_kind, file_path, ALUMNFORCE_GROUP_FIELDS): parse_reports_this_kind.append(parse_report) if not value: continue - value['last_update'] = file_date - models.Group.objects.update_or_create(af_id=value['af_id'], defaults=value) + value["last_update"] = file_date + models.Group.objects.update_or_create(af_id=value["af_id"], defaults=value) num_values += 1 elif file_kind == "groupmembers": num_values = 0 - for (parse_report, value) in load_csv(file_kind, file_path, ALUMNFORCE_GROUPMEMBER_FIELDS): + for parse_report, value in load_csv(file_kind, file_path, ALUMNFORCE_GROUPMEMBER_FIELDS): parse_reports_this_kind.append(parse_report) if not value: continue try: - account = models.Account.objects.get(af_id=value['user_id']) + account = models.Account.objects.get(af_id=value["user_id"]) except models.Account.DoesNotExist: self.log_warning( - file_date, file_kind, + file_date, + file_kind, "Unable to find user with AF ID {} (AX ID {})".format( - value['user_id'], repr(value['user_ax_id']) - )) + value["user_id"], repr(value["user_ax_id"]) + ), + ) continue try: - group = models.Group.objects.get(af_id=value['group_id']) + group = models.Group.objects.get(af_id=value["group_id"]) except models.Group.DoesNotExist: self.log_warning( - file_date, file_kind, - "Unable to find group with AF ID {}".format( - value['group_id'] - )) + file_date, file_kind, "Unable to find group with AF ID {}".format(value["group_id"]) + ) continue try: - role = ALUMNFORCE_GROUPMEMBER_ROLES[value['role']] + role = ALUMNFORCE_GROUPMEMBER_ROLES[value["role"]] except KeyError: self.log_warning( - file_date, file_kind, - "Unable to find group role {}".format(repr(value['role'])) + file_date, file_kind, "Unable to find group role {}".format(repr(value["role"])) ) continue models.GroupMembership.objects.update_or_create( account=account, group=group, - defaults={'role': role, 'last_update': file_date}, + defaults={"role": role, "last_update": file_date}, ) num_values += 1 else: @@ -541,14 +528,14 @@ def handle(self, *args, **options): reports_by_afid = {} for parse_report in parse_reports_this_kind: - reports_by_afid.setdefault(parse_report['af_id'], []).append(parse_report) + reports_by_afid.setdefault(parse_report["af_id"], []).append(parse_report) # Update records problem_changes_this_file = {} # These are not all users, only users referred to by imported data. - for (af_id, reports) in reports_by_afid.items(): + for af_id, reports in reports_by_afid.items(): try: account = models.Account.objects.get(af_id=af_id) account_label_for_filename = account.xorg_id @@ -573,15 +560,19 @@ def handle(self, *args, **options): if user_is_affected: print( - f"Recording current problem on user {account_label_for_content} with kind {file_kind}: {current_problem_file_path}") + f"Recording current problem on user {account_label_for_content} with kind {file_kind}: " + f"{current_problem_file_path}" + ) with open(current_problem_file_path, "a") as rej_file: rej_file.write( - f"### Soucis de type {file_kind} concernant le compte {account_label_for_content}\n\n") + f"### Soucis de type {file_kind} concernant le compte {account_label_for_content}\n\n" + ) for report in user_reports_with_problem: rej_file.write( "------------------------------------------------------------------------\n" + "\n".join("{:<10}: {}".format(k, v) for k, v in report.items()) - + "------------------------------------------------------------------------\n") + + "------------------------------------------------------------------------\n" + ) else: if user_was_affected: print(f"Deleting rejection file: {current_problem_file_path}") @@ -602,17 +593,24 @@ def handle(self, *args, **options): lines_to_log = reports for report in lines_to_log: problem_archive_file_path = compute_problem_archive_file_path( - file_kind, af_id, file_path, problem_archive_file_marker, - account_label_for_filename, report["line_hash"]) + file_kind, + af_id, + file_path, + problem_archive_file_marker, + account_label_for_filename, + report["line_hash"], + ) print(f"Recording to problem archive: {problem_archive_file_path}") if problem_archive_file_marker == "resolved": resolved_to_be_also_in_report.append( - (account_label_for_filename, problem_archive_file_path)) + (account_label_for_filename, problem_archive_file_path) + ) with open(problem_archive_file_path, "w") as rej_file: rej_file.write( "\n------------------------------------------------------------------------\n" + "\n".join("{:<10}: {}".format(k, v) for k, v in report.items()) - + "\n------------------------------------------------------------------------\n") + + "\n------------------------------------------------------------------------\n" + ) # Here finished importing and processing one file, now reporting @@ -622,8 +620,7 @@ def handle(self, *args, **options): users_in_this_case = problem_changes_this_file.get(case_number) # print (f"For case {case_number} users: {users_in_this_case}") if users_in_this_case: - case = ["ras", "nouveau souci", "souci résolu", - "souci répété"][case_number] + case = ["ras", "nouveau souci", "souci résolu", "souci répété"][case_number] if case_number == 0: pass # report_by_file_then_user.append(f" {case} pour {len(users_in_this_case)} utilisateur(s):") @@ -649,24 +646,29 @@ def handle(self, *args, **options): "Ce message est envoyé par le sous-système qui importe les dernières données d'annuaires depuis l'AX.", "", "## Résumé du traitement", - "" + "", ] # import_report_lines += [f"Nombre de fichiers à importer : {len(options['csvfile'])}, liste ci-dessous:", ""] - import_report_lines += ["Importé " + os.path.basename(n) for n in options['csvfile']] + import_report_lines += ["Importé " + os.path.basename(n) for n in options["csvfile"]] # import_report_lines += [f"Nombre de fichiers à importer : {len(options['csvfile'])}."] import_report_lines += report_by_file_then_user import_report_lines += [ - "", "## Synthèse des utilisateurs affectés par des soucis", "", + "", + "## Synthèse des utilisateurs affectés par des soucis", + "", "Cette section ne se limite pas aux nouveautés de la dernière importation.", - "Elle considère tous les incidents non encore résolus.", "", + "Elle considère tous les incidents non encore résolus.", + "", "Les erreurs de données manifestes (exemple : tabulation dans un champ, année sur 3 chiffres)", - "sont à corriger dans la base en amont. Alors l'entrée correspondante disparaîtra à la prochaine importation.", - "",] + "sont à corriger dans la base en amont. Alors l'entrée correspondante disparaîtra à la prochaine importation.", # noqa: E501 + "", + ] from pathlib import Path + rejects_directory = Path(settings.PERSISTENT_DIRECTORY) / "current_problems_by_id" active_rejections = [] @@ -688,22 +690,25 @@ def handle(self, *args, **options): import_report_lines.append("\n## Détails des utilisateurs affectés") - for (kind, name, rejfile) in active_rejections: + for kind, name, rejfile in active_rejections: with rejfile.open() as f: import_report_lines.append("\n" + "".join(f.readlines())) if resolved_to_be_also_in_report: import_report_lines += [ - "", "## Détails des cas résolus", "", + "", + "## Détails des cas résolus", + "", "Certaines tables peuvent avoir plusieurs lignes par utilisateur (typiquement userjobs).", "Le cas est résolu quand toutes les lignes sont valides.", - "Par simplicité on montre ici toutes les lignes qui concernent l'utilisateur.", "" + "Par simplicité on montre ici toutes les lignes qui concernent l'utilisateur.", + "", ] - for (account_label_for_filename, resolved_file_path) in resolved_to_be_also_in_report: + for account_label_for_filename, resolved_file_path in resolved_to_be_also_in_report: import_report_lines.append(f"Camarade {account_label_for_filename}") import_report_lines.append(f"Fichier {resolved_file_path}") - with open(resolved_file_path, 'r', encoding='utf-8') as f: + with open(resolved_file_path, "r", encoding="utf-8") as f: import_report_lines.append("".join(f.readlines())) # All report info is gathered. Assemble that into an e-mail body. @@ -712,39 +717,43 @@ def handle(self, *args, **options): human_labels_for_cases = [None, "nouveau(x)", "résolu(s)", "répété(s)"] - set_of_affected_user_changes = [f"{human_labels_for_cases[case]} pour {len(affected_users)} camarade(s)" - for (case, affected_users) in problem_changes_all_files.items() if case != 0 - ] + set_of_affected_user_changes = [ + f"{human_labels_for_cases[case]} pour {len(affected_users)} camarade(s)" + for (case, affected_users) in problem_changes_all_files.items() + if case != 0 + ] - set_of_affected_user_changes_text = ", ".join( - set_of_affected_user_changes) if set_of_affected_user_changes else "aucun changement" + set_of_affected_user_changes_text = ( + ", ".join(set_of_affected_user_changes) if set_of_affected_user_changes else "aucun changement" + ) - one_line_subject = f"Souci(s) d'importation : {set_of_affected_user_changes_text}, bilan {len(active_rejections)}" + one_line_subject = ( + f"Souci(s) d'importation : {set_of_affected_user_changes_text}, bilan {len(active_rejections)}" + ) worth_an_email = bool(set_of_affected_user_changes) or bool(active_rejections) - report_as_text = '\n'.join([ - "------------------------------------------------------------------------", - "Vaut un e-mail ? " + ["non", "oui"][worth_an_email], - "------------------------------------------------------------------------", - one_line_subject, - "------------------------------------------------------------------------", - overall_report_text - ]) + report_as_text = "\n".join( + [ + "------------------------------------------------------------------------", + "Vaut un e-mail ? " + ["non", "oui"][worth_an_email], + "------------------------------------------------------------------------", + one_line_subject, + "------------------------------------------------------------------------", + overall_report_text, + ] + ) timestamp_start_str = timestamp_start.strftime("%Yy%mm%dd-%Hh%Mm%S.%fs") report_file_name = timestamp_start_str - if len(options['csvfile']) == 1: - report_file_name += "_for_file_" + os.path.basename(options['csvfile'][0]) + if len(options["csvfile"]) == 1: + report_file_name += "_for_file_" + os.path.basename(options["csvfile"][0]) report_file_name += ".report.txt" - directory = os.path.join( - settings.PERSISTENT_DIRECTORY, - timestamp_start.strftime("reports/%Y") - ) + directory = os.path.join(settings.PERSISTENT_DIRECTORY, timestamp_start.strftime("reports/%Y")) os.makedirs(directory, exist_ok=True) report_full_path = os.path.join(directory, report_file_name) @@ -756,8 +765,5 @@ def handle(self, *args, **options): if worth_an_email: send_mail( - settings.EMAIL_SUBJECT_PREFIX + one_line_subject, - overall_report_text, - None, - settings.REPORT_RECIPIENTS + settings.EMAIL_SUBJECT_PREFIX + one_line_subject, overall_report_text, None, settings.REPORT_RECIPIENTS ) diff --git a/xorgdata/alumnforce/migrations/0015_alter_academicinformation_diplomed_and_more.py b/xorgdata/alumnforce/migrations/0015_alter_academicinformation_diplomed_and_more.py new file mode 100644 index 0000000..80a6595 --- /dev/null +++ b/xorgdata/alumnforce/migrations/0015_alter_academicinformation_diplomed_and_more.py @@ -0,0 +1,68 @@ +# Generated by Django 5.2.13 on 2026-05-03 13:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('alumnforce', '0014_add_exportlog_table'), + ] + + operations = [ + migrations.AlterField( + model_name='academicinformation', + name='diplomed', + field=models.BooleanField(blank=True, null=True), + ), + migrations.AlterField( + model_name='academicinformation', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='account', + name='address_npai', + field=models.BooleanField(blank=True, null=True), + ), + migrations.AlterField( + model_name='account', + name='dead', + field=models.BooleanField(blank=True, null=True), + ), + migrations.AlterField( + model_name='exportlog', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='groupmembership', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='importlog', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='professionnalinformation', + name='buyer_of_company', + field=models.BooleanField(blank=True, null=True), + ), + migrations.AlterField( + model_name='professionnalinformation', + name='creator_of_company', + field=models.BooleanField(blank=True, null=True), + ), + migrations.AlterField( + model_name='professionnalinformation', + name='current', + field=models.BooleanField(blank=True, null=True), + ), + migrations.AlterField( + model_name='professionnalinformation', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/xorgdata/alumnforce/models.py b/xorgdata/alumnforce/models.py index 2fe9822..8032a6c 100644 --- a/xorgdata/alumnforce/models.py +++ b/xorgdata/alumnforce/models.py @@ -1,6 +1,6 @@ from django.core.validators import validate_comma_separated_integer_list from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from xorgdata.utils.fields import DottedSlugField, UnboundedCharField @@ -62,7 +62,7 @@ class Account(models.Model): address_city = UnboundedCharField(blank=True) address_state = UnboundedCharField(blank=True) address_country = models.CharField(max_length=2, blank=True) - address_npai = models.NullBooleanField() + address_npai = models.BooleanField(null=True, blank=True) phone_personnal = UnboundedCharField(blank=True) phone_mobile = UnboundedCharField(blank=True) email_1 = models.EmailField() @@ -70,12 +70,11 @@ class Account(models.Model): nationality = UnboundedCharField(blank=True) nationality_2 = UnboundedCharField(blank=True) nationality_3 = UnboundedCharField(blank=True) - dead = models.NullBooleanField() + dead = models.BooleanField(null=True, blank=True) deathdate = models.DateField(blank=True, null=True) dead_for_france = UnboundedCharField(blank=True) user_kind = models.IntegerField() - additional_roles = UnboundedCharField(blank=True, - validators=[validate_comma_separated_integer_list]) + additional_roles = UnboundedCharField(blank=True, validators=[validate_comma_separated_integer_list]) xorg_id = DottedSlugField(max_length=255, blank=True, null=True) school_id = UnboundedCharField(blank=True) admission_path = UnboundedCharField(blank=True) @@ -99,35 +98,35 @@ def __str__(self): if self.xorg_id: result = self.xorg_id elif self.first_name and self.last_name: - result = '{} {}'.format(self.first_name, self.last_name) + result = "{} {}".format(self.first_name, self.last_name) elif self.email_1: result = self.email_1 else: - result = '?' + result = "?" # Add the AX ID or the AF ID if self.ax_id: - result += ' (AX ID {})'.format(self.ax_id) + result += " (AX ID {})".format(self.ax_id) else: - result += ' (AF ID {})'.format(self.af_id) + result += " (AF ID {})".format(self.af_id) return result def get_additional_roles(self): """Return the additional roles as a list of integers""" if not self.additional_roles: return [] - return [int(r) for r in self.additional_roles.split(',')] + return [int(r) for r in self.additional_roles.split(",")] @property def alumnforce_profile_url(self): """URL on AlumnForce website""" - return 'https://ax.polytechnique.org/person/by-id/{:d}'.format(self.af_id) + return "https://ax.polytechnique.org/person/by-id/{:d}".format(self.af_id) class AcademicInformation(models.Model): - account = models.ForeignKey('Account', related_name='degrees', on_delete=models.CASCADE) + account = models.ForeignKey("Account", related_name="degrees", on_delete=models.CASCADE) diploma_reference = UnboundedCharField() - diplomed = models.NullBooleanField() + diplomed = models.BooleanField(null=True, blank=True) diplomation_date = models.DateField(blank=True, null=True) cycle = UnboundedCharField(blank=True) domain = UnboundedCharField(blank=True) @@ -136,7 +135,7 @@ class AcademicInformation(models.Model): class ProfessionnalInformation(models.Model): - account = models.ForeignKey('Account', related_name='jobs', on_delete=models.CASCADE) + account = models.ForeignKey("Account", related_name="jobs", on_delete=models.CASCADE) title = UnboundedCharField() role = UnboundedCharField(blank=True) company_name = UnboundedCharField() @@ -156,9 +155,9 @@ class ProfessionnalInformation(models.Model): start_date = models.DateField(blank=True, null=True) end_date = models.DateField(blank=True, null=True) contract_kind = UnboundedCharField(blank=True) - current = models.NullBooleanField() - creator_of_company = models.NullBooleanField() - buyer_of_company = models.NullBooleanField() + current = models.BooleanField(null=True, blank=True) + creator_of_company = models.BooleanField(null=True, blank=True) + buyer_of_company = models.BooleanField(null=True, blank=True) last_update = models.DateField() @@ -177,23 +176,23 @@ def __str__(self): class GroupMembership(models.Model): # Use memership values defined by Alumnforce MEMBERSHIP_ROLES = ( - ('banned', _('banned')), - ('invited', _('invited')), - ('member', _('member')), - ('moderator', _('moderator')), - ('onlist', _('on list')), - ('responsible', _('responsible')), - ('unsubscribed', _('unsubscribed')), + ("banned", _("banned")), + ("invited", _("invited")), + ("member", _("member")), + ("moderator", _("moderator")), + ("onlist", _("on list")), + ("responsible", _("responsible")), + ("unsubscribed", _("unsubscribed")), ) # Roles that really mean that the user belongs to the group - IN_GROUP_ROLES = frozenset(('member', 'moderator', 'onlist', 'responsible')) - account = models.ForeignKey(Account, related_name='group_memberships', on_delete=models.CASCADE) - group = models.ForeignKey(Group, related_name='memberships', on_delete=models.CASCADE) + IN_GROUP_ROLES = frozenset(("member", "moderator", "onlist", "responsible")) + account = models.ForeignKey(Account, related_name="group_memberships", on_delete=models.CASCADE) + group = models.ForeignKey(Group, related_name="memberships", on_delete=models.CASCADE) role = models.SlugField(choices=MEMBERSHIP_ROLES) last_update = models.DateField() class Meta: - unique_together = ('group', 'account') + unique_together = ("group", "account") class ImportLog(models.Model): @@ -201,20 +200,20 @@ class ImportLog(models.Model): # parse incoming export files: users need to be first created, then groups, # then everything else (that depends on users and/or groups). KNOWN_EXPORT_KINDS = ( - ('users', _('users')), - ('groups', _('groups')), - ('groupmembers', _('groupmembers')), - ('userdegrees', _('userdegrees')), - ('userjobs', _('userjobs')), + ("users", _("users")), + ("groups", _("groups")), + ("groupmembers", _("groupmembers")), + ("userdegrees", _("userdegrees")), + ("userjobs", _("userjobs")), ) # Error code when importing data SUCCESS = 0 ALUMNFORCE_ERROR = 1 XORG_ERROR = 2 ERROR_CODES = ( - (SUCCESS, _('success')), - (ALUMNFORCE_ERROR, _('AlumnForce error')), - (XORG_ERROR, _('X.org error')), + (SUCCESS, _("success")), + (ALUMNFORCE_ERROR, _("AlumnForce error")), + (XORG_ERROR, _("X.org error")), ) date = models.DateField() export_kind = models.SlugField(choices=KNOWN_EXPORT_KINDS) @@ -225,14 +224,10 @@ class ImportLog(models.Model): class ExportLog(models.Model): - KIND_AUTH = 'auth' - KNOWN_KINDS = ( - (KIND_AUTH, _('X.org auth')), - ) + KIND_AUTH = "auth" + KNOWN_KINDS = ((KIND_AUTH, _("X.org auth")),) SUCCESS = 0 - ERROR_CODES = ( - (SUCCESS, _('success')), - ) + ERROR_CODES = ((SUCCESS, _("success")),) date = models.DateField() export_kind = models.SlugField(choices=KNOWN_KINDS) error = models.IntegerField(choices=ERROR_CODES) diff --git a/xorgdata/alumnforce/views.py b/xorgdata/alumnforce/views.py index 2fc2484..a079b85 100644 --- a/xorgdata/alumnforce/views.py +++ b/xorgdata/alumnforce/views.py @@ -8,34 +8,34 @@ class SummaryView(TemplateView): - template_name = 'xorgdata/summary.html' + template_name = "xorgdata/summary.html" def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) # Get the last import logs from the database, for each defined kind last_logs = [] for kind, _kind_name in models.ImportLog.KNOWN_EXPORT_KINDS: - qs = models.ImportLog.objects.filter(export_kind=kind).order_by('-date', '-is_incremental') + qs = models.ImportLog.objects.filter(export_kind=kind).order_by("-date", "-is_incremental") try: last_logs.append(qs[:1].get()) except models.ImportLog.DoesNotExist: pass - context['last_imp_logs_by_kind'] = last_logs + context["last_imp_logs_by_kind"] = last_logs # Get the last export logs from the database, for each defined kind last_logs = [] for kind, _kind_name in models.ExportLog.KNOWN_KINDS: - qs = models.ExportLog.objects.filter(export_kind=kind).order_by('-date') + qs = models.ExportLog.objects.filter(export_kind=kind).order_by("-date") try: last_logs.append(qs[:1].get()) except models.ExportLog.DoesNotExist: pass - context['last_exp_logs_by_kind'] = last_logs + context["last_exp_logs_by_kind"] = last_logs return context class IssuesView(UserPassesTestMixin, TemplateView): - template_name = 'xorgdata/issues.html' + template_name = "xorgdata/issues.html" def test_func(self): """Restrict this view to the superuser""" @@ -45,7 +45,7 @@ def find_issues(self, account): """Find issues in an account""" account_issues = [] - if account.civility not in ('', 'Mme', 'M'): + if account.civility not in ("", "Mme", "M"): account_issues.append("Unknown civility {}".format(repr(account.civility))) if account.user_kind not in models.Account.KINDS: @@ -62,7 +62,7 @@ def find_issues(self, account): if account.xorg_id: # Verify the format of X.org ID - if re.search(r'[^a-z0-9.-]', account.xorg_id): + if re.search(r"[^a-z0-9.-]", account.xorg_id): account_issues.append("Invalid X.org ID: {}".format(repr(account.xorg_id))) else: # Check whether the user should have a X.org ID @@ -72,7 +72,7 @@ def find_issues(self, account): account_issues.append("Missing X.org ID for student") # Verify email addresses against a simple regular expression - email_regexp = re.compile(r'^[a-zA-Z0-9._+-]+@[a-zA-Z0-9.-]+$') + email_regexp = re.compile(r"^[a-zA-Z0-9._+-]+@[a-zA-Z0-9.-]+$") if account.email_1 and not email_regexp.match(account.email_1): account_issues.append("Invalid email address 1 {}".format(repr(account.email_1))) if account.email_2 and not email_regexp.match(account.email_2): @@ -84,30 +84,27 @@ def get_context_data(self, **kwargs): # Find accounts with duplicate IDs duplicated_ax_id = { - row['ax_id']: row['count'] - for row in models.Account.objects - .filter(deleted_since=None) + row["ax_id"]: row["count"] + for row in models.Account.objects.filter(deleted_since=None) .exclude(ax_id=None) - .values('ax_id') - .annotate(count=Count('af_id')) + .values("ax_id") + .annotate(count=Count("af_id")) .filter(count__gt=1) } duplicated_xorg_id = { - row['xorg_id']: row['count'] - for row in models.Account.objects - .filter(deleted_since=None) + row["xorg_id"]: row["count"] + for row in models.Account.objects.filter(deleted_since=None) .exclude(xorg_id=None) - .values('xorg_id') - .annotate(count=Count('af_id')) + .values("xorg_id") + .annotate(count=Count("af_id")) .filter(count__gt=1) } duplicated_school_id = { - row['school_id']: row['count'] - for row in models.Account.objects - .filter(deleted_since=None) - .exclude(school_id='') - .values('school_id') - .annotate(count=Count('af_id')) + row["school_id"]: row["count"] + for row in models.Account.objects.filter(deleted_since=None) + .exclude(school_id="") + .values("school_id") + .annotate(count=Count("af_id")) .filter(count__gt=1) } @@ -116,23 +113,28 @@ def get_context_data(self, **kwargs): account_issues = self.find_issues(account) dup_count = duplicated_ax_id.get(account.ax_id) if dup_count is not None: - account_issues.append("Duplicated AX ID {}, shared with {} accounts".format( - repr(account.ax_id), dup_count)) + account_issues.append( + "Duplicated AX ID {}, shared with {} accounts".format(repr(account.ax_id), dup_count) + ) dup_count = duplicated_xorg_id.get(account.xorg_id) if dup_count is not None: - account_issues.append("Duplicated X.org ID {}, shared with {} accounts".format( - repr(account.xorg_id), dup_count)) + account_issues.append( + "Duplicated X.org ID {}, shared with {} accounts".format(repr(account.xorg_id), dup_count) + ) dup_count = duplicated_school_id.get(account.school_id) if dup_count is not None: - account_issues.append("Duplicated school ID {}, shared with {} accounts".format( - repr(account.school_id), dup_count)) + account_issues.append( + "Duplicated school ID {}, shared with {} accounts".format(repr(account.school_id), dup_count) + ) if account_issues: - issues.append({ - 'account': account, - 'issues': account_issues, - }) - context['issues'] = issues + issues.append( + { + "account": account, + "issues": account_issues, + } + ) + context["issues"] = issues return context diff --git a/xorgdata/settings.py b/xorgdata/settings.py index bd81e8e..8e54786 100644 --- a/xorgdata/settings.py +++ b/xorgdata/settings.py @@ -10,154 +10,155 @@ https://docs.djangoproject.com/en/2.0/ref/settings/ """ -import getconf import os +import getconf from django.core.exceptions import ImproperlyConfigured - # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -config = getconf.ConfigGetter('xorgdata', [ - '/etc/xorgdata/*.ini', - os.path.join(BASE_DIR, 'local_settings.ini'), -]) +config = getconf.ConfigGetter( + "xorgdata", + [ + "/etc/xorgdata/*.ini", + os.path.join(BASE_DIR, "local_settings.ini"), + ], +) -APPMODE = config.getstr('app.mode', 'dev') -assert APPMODE in ('dev', 'dist', 'prod'), "Invalid application mode %s" % APPMODE +APPMODE = config.getstr("app.mode", "dev") +assert APPMODE in ("dev", "dist", "prod"), "Invalid application mode %s" % APPMODE # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = config.getstr('app.secret_key', 'Dev only!!') +SECRET_KEY = config.getstr("app.secret_key", "Dev only!!") # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = config.getbool('app.debug', APPMODE == 'dev') +DEBUG = config.getbool("app.debug", APPMODE == "dev") -if config.getstr('site.admin_mail'): - ADMINS = ( - ("XorgData admins", config.getstr('site.admin_mail')), - ) +if config.getstr("site.admin_mail"): + ADMINS = (("XorgData admins", config.getstr("site.admin_mail")),) -ALLOWED_HOSTS = config.getlist('site.allowed_hosts', []) +ALLOWED_HOSTS = config.getlist("site.allowed_hosts", []) # Application definition INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'xorgdata', - 'xorgdata.alumnforce' + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "xorgdata", + "xorgdata.alumnforce", ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = 'xorgdata.urls' +ROOT_URLCONF = "xorgdata.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, ] -WSGI_APPLICATION = 'xorgdata.wsgi.application' +WSGI_APPLICATION = "xorgdata.wsgi.application" # Database # https://docs.djangoproject.com/en/2.0/ref/settings/#databases _ENGINE_MAP = { - 'sqlite': 'django.db.backends.sqlite3', - 'mysql': 'django.db.backends.mysql', - 'postgresql': 'django.db.backends.postgresql_psycopg2', + "sqlite": "django.db.backends.sqlite3", + "mysql": "django.db.backends.mysql", + "postgresql": "django.db.backends.postgresql_psycopg2", } -_engine = config.getstr('db.engine', 'sqlite') +_engine = config.getstr("db.engine", "sqlite") if _engine not in _ENGINE_MAP: raise ImproperlyConfigured( - "DB engine %s is unknown; please choose from %s" % - (_engine, ', '.join(sorted(_ENGINE_MAP.keys()))) + "DB engine %s is unknown; please choose from %s" % (_engine, ", ".join(sorted(_ENGINE_MAP.keys()))) ) -if _engine == 'sqlite': - if APPMODE == 'dev': - _default_db_name = os.path.join(BASE_DIR, 'dev', 'db.sqlite') +if _engine == "sqlite": + if APPMODE == "dev": + _default_db_name = os.path.join(BASE_DIR, "dev", "db.sqlite") else: - _default_db_name = '/var/lib/xorgdata/db.sqlite' + _default_db_name = "/var/lib/xorgdata/db.sqlite" else: - _default_db_name = 'xorgdata' + _default_db_name = "xorgdata" DATABASES = { - 'default': { - 'ENGINE': _ENGINE_MAP[_engine], - 'NAME': config.getstr('db.name', _default_db_name), - 'USER': config.getstr('db.user'), - 'PASSWORD': config.getstr('db.password'), - 'HOST': config.getstr('db.host', 'localhost'), - 'PORT': config.getstr('db.port'), + "default": { + "ENGINE": _ENGINE_MAP[_engine], + "NAME": config.getstr("db.name", _default_db_name), + "USER": config.getstr("db.user"), + "PASSWORD": config.getstr("db.password"), + "HOST": config.getstr("db.host", "localhost"), + "PORT": config.getstr("db.port"), }, } -if _engine == 'mysql': +if _engine == "mysql": # Detect data integrity problems in MySQL early # https://django-mysql.readthedocs.io/en/latest/checks.html#django-mysql-w001-strict-mode - DATABASES['default']['OPTIONS'] = { - 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'", + DATABASES["default"]["OPTIONS"] = { + "init_command": "SET sql_mode='STRICT_TRANS_TABLES'", } +# Default primary key field type. +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" # Password validation # https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] # Use Django admin for authentication -LOGIN_URL = '/admin/login/' +LOGIN_URL = "/admin/login/" # Internationalization # https://docs.djangoproject.com/en/2.0/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True @@ -169,8 +170,8 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/2.0/howto/static-files/ -STATIC_URL = '/static/' -STATIC_ROOT = os.path.join(BASE_DIR, 'static') +STATIC_URL = "/static/" +STATIC_ROOT = os.path.join(BASE_DIR, "static") EMAIL_HOST = config.getstr("email.host") EMAIL_PORT = config.getint("email.port") @@ -179,19 +180,19 @@ EMAIL_USE_TLS = config.getbool("email.tls") DEFAULT_FROM_EMAIL = config.getstr("email.default_from_email", "Polytechnique.org ") SERVER_EMAIL = config.getstr("email.server_email", "Polytechnique.org ") -EMAIL_SUBJECT_PREFIX = config.getstr('email.subject_prefix', '[xorgdata]') + ' ' -REPORT_RECIPIENTS =config.getstr('email.report_recipients', '').split(",") +EMAIL_SUBJECT_PREFIX = config.getstr("email.subject_prefix", "[xorgdata]") + " " +REPORT_RECIPIENTS = config.getstr("email.report_recipients", "").split(",") # In development mode, send messages to the console -if APPMODE == 'dev': - EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +if APPMODE == "dev": + EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" # Security SECURE_CONTENT_TYPE_NOSNIFF = True SECURE_BROWSER_XSS_FILTER = True -X_FRAME_OPTIONS = 'DENY' +X_FRAME_OPTIONS = "DENY" -USE_HTTPS = (APPMODE == 'prod') or config.getbool("security.use_ssl") +USE_HTTPS = (APPMODE == "prod") or config.getbool("security.use_ssl") SECURE_SSL_REDIRECT = USE_HTTPS SESSION_COOKIE_SECURE = USE_HTTPS CSRF_COOKIE_SECURE = USE_HTTPS @@ -200,7 +201,7 @@ # Force using HSTS with HTTPS SECURE_HSTS_INCLUDE_SUBDOMAINS = True SECURE_HSTS_PRELOAD = True - SECURE_HSTS_SECONDS = config.getint('security.hsts_seconds', 15768000) + SECURE_HSTS_SECONDS = config.getint("security.hsts_seconds", 15768000) # All data that needs to be preserved between and beyond runs PERSISTENT_DIRECTORY = config.getstr("persistence.root_path", "/tmp") @@ -210,7 +211,9 @@ ALUMNFORCE_FTP_USER = config.getstr("alumnforce_ftp.user") ALUMNFORCE_FTP_PASSWORD = config.getstr("alumnforce_ftp.password") ALUMNFORCE_FTP_REMOTE_DIRECTORY = config.getstr("alumnforce_ftp.remote_directory", "export") -ALUMNFORCE_FTP_LOCAL_DIRECTORY = config.getstr("alumnforce_ftp.local_directory", os.path.join(PERSISTENT_DIRECTORY, "xorgdata-download")) +ALUMNFORCE_FTP_LOCAL_DIRECTORY = config.getstr( + "alumnforce_ftp.local_directory", os.path.join(PERSISTENT_DIRECTORY, "xorgdata-download") +) # Settings for the xorgauth API which receives data XORGAUTH_HOST = config.getstr("xorgauth.host", "auth.polytechnique.org") diff --git a/xorgdata/urls.py b/xorgdata/urls.py index d576b0f..157a367 100644 --- a/xorgdata/urls.py +++ b/xorgdata/urls.py @@ -13,16 +13,16 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ + from django.contrib import admin from django.urls import path from django.views.generic import TemplateView import xorgdata.alumnforce.views - urlpatterns = [ - path('admin/', admin.site.urls), - path('robots.txt', TemplateView.as_view(template_name='robots.txt', content_type='text/plain'), name='robots'), - path('', xorgdata.alumnforce.views.SummaryView.as_view(), name='index'), - path('issues', xorgdata.alumnforce.views.IssuesView.as_view(), name='issues'), + path("admin/", admin.site.urls), + path("robots.txt", TemplateView.as_view(template_name="robots.txt", content_type="text/plain"), name="robots"), + path("", xorgdata.alumnforce.views.SummaryView.as_view(), name="index"), + path("issues", xorgdata.alumnforce.views.IssuesView.as_view(), name="issues"), ] diff --git a/xorgdata/utils/fields.py b/xorgdata/utils/fields.py index 0a57faa..0353c8c 100644 --- a/xorgdata/utils/fields.py +++ b/xorgdata/utils/fields.py @@ -3,10 +3,10 @@ # This code is distributed under the Affero General Public License version 3 import re +from django import forms from django.core.validators import RegexValidator from django.db import models -from django import forms -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class UnboundedCharField(models.TextField): @@ -16,23 +16,25 @@ class UnboundedCharField(models.TextField): Like the standard :class:`~django.db.models.fields.CharField` widget, a ``select`` widget is automatically used if the field defines ``choices``. """ + def __init__(self, *args, **kwargs): - if kwargs.get('unique'): + if kwargs.get("unique"): raise ValueError("UnboundedCharField can not be 'unique' as this is not supported by MySQL") return super(UnboundedCharField, self).__init__(*args, **kwargs) def formfield(self, **kwargs): - kwargs['widget'] = None if self.choices else forms.TextInput + kwargs["widget"] = None if self.choices else forms.TextInput return super(UnboundedCharField, self).formfield(**kwargs) validate_dotted_slug = RegexValidator( - re.compile(r'^[-a-zA-Z0-9_.]+\Z'), + re.compile(r"^[-a-zA-Z0-9_.]+\Z"), _("Enter a valid 'slug' consisting of letters, numbers, underscores, dots or hyphens."), - 'invalid' + "invalid", ) class DottedSlugField(models.CharField): """Slug field which allows dot""" + default_validators = [validate_dotted_slug]