collect-datasets #2189

	name: collect-datasets

	on:
	schedule:
	# Run at 0 minute past every 12th hour
	- cron: '0 /12 * *'
	push:
	branches: [ master ]
	pull_request:
	branches: [ master ]
	workflow_dispatch:

	jobs:
	collect-datasets:
	runs-on: ubuntu-latest

	steps:
	- uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.10'

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install flake8 pytest
	if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

	- name: Lint with flake8
	run: \|
	# stop the build if there are Python syntax errors or undefined names
	flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
	# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
	flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

	- name: Collect v1 and v2 datasets from Metadata Catalogue
	run: \|
	python datasets.py

	- name: Perform v1 completeness and schema validation checks
	run: \|
	python quality_checks.py

	- name: Perform V2 completeness and schema validation checks
	run: \|
	python quality_checks_v2.py

	- name: Merge V1 and V2 quality scores
	run: \|
	python merge_quality_scores.py

	- name: Auto-generate README
	run: \|
	python update_readme.py

	- uses: stefanzweifel/git-auto-commit-action@v5
	with:
	commit_message: Commit new datasets
	branch: ${{ github.head_ref }}
	commit_user_name: Susheel Varma
	commit_user_email: susheel.varma@hdruk.ac.uk
	commit_author: Susheel Varma <susheel.varma@hdruk.ac.uk>

Provide feedback