Merge pull request #502 from hotosm/feat/visibility-and-anon-reads

Feature : visibility on the models

Author: kshitijrajsharma

backend/ARCHITECTURE.md

@@ -23,7 +23,7 @@ What happens behind each step:
 | Step | What the backend does | Output |
 | --- | --- | --- |
 | **1. AOI** | Validates polygon, stores in postgres. | Row in `datasets_aoi`. |
-| **2. Build dataset** | Async worker downloads OAM tiles + OSM labels for the AOI, uploads chips + `labels.geojson` to MinIO, registers a STAC item under `datasets/`. | Published STAC dataset. Poll `GET /datasets/{id}/` until `build_status=published`. |
+| **2. Build dataset** | Async worker downloads OAM tiles + OSM labels for the AOI, uploads chips + `labels.geojson` to MinIO, registers a STAC item under `datasets/`. | Built STAC dataset. Poll `GET /datasets/{id}/` until `status=built`. |
 | **3. Submit training** | Async worker submits a ZenML pipeline. ZenML schedules an orchestrator + step pods on the autoscaling ml pool (`split -> train -> eval -> onnx`). Worker polls ZenML status into the DB every 30s. | Trained `weights.pt` + `model.onnx` in MinIO. Poll `GET /trainings/{id}/` until `status=completed`. Tail with `GET /trainings/runs/{run_id}/logs/`. |
 | **4. Promote** | API builds a versioned STAC `local-models/` item from the run's hyperparameters + asset URLs, validates against the base-model's `fair:hyperparameters_spec`, publishes. | `local_model_stac_id`. |
 | **5. Predict** | Async worker downloads chips for the requested bbox, submits an inference pipeline, then post-processes the geojson into `.fgb` + `.pmtiles` via tippecanoe. | Three presigned URLs at `GET /predictions/{id}/result/` once `results_ready=true`. |
@@ -69,7 +69,8 @@ erDiagram
         string stac_id UK "STAC item id under datasets/"
         string title
         url source_imagery "TMS template"
-        string build_status "draft|building|published|failed"
+        string status "draft|building|built|failed"
+        string visibility "private|public"
         bigint user_id FK "-> OsmUser.osm_id"
         datetime created_at
         datetime last_modified
@@ -86,7 +87,8 @@ erDiagram
     LocalModel {
         int id PK
         string name UK "= ZenML model_name = STAC mlm:name"
-        string status "draft|published|archived"
+        string status "active|archived"
+        string visibility "private|public"
         bigint user_id FK
         datetime created_at
     }
@@ -116,7 +118,7 @@ erDiagram
         smallint zoom
         json params "confidence_threshold, ..."
         bool remove_osm
-        bool is_public
+        string visibility "private|public"
         text description
         string status
         bool results_ready "geojson+fgb+pmtiles materialized"
@@ -130,7 +132,7 @@ erDiagram
 
 ## 3. API reference
 
-All endpoints live under `/api/v1/` and require `Authorization: Bearer <token>`. The token is `FAIR_DEV_TOKEN` when `AUTH_PROVIDER=dev`, or a Hanko-issued JWT when `AUTH_PROVIDER=hanko`. Swagger UI: `/api/docs/`. OpenAPI: `/api/schema/`.
+All endpoints live under `/api/v1/`. Writes and owner-scoped reads require `Authorization: Bearer <token>`. The token is `FAIR_DEV_TOKEN` when `AUTH_PROVIDER=dev`, or a Hanko-issued JWT when `AUTH_PROVIDER=hanko`. `GET` on datasets, local-models, and predictions is open anonymously for rows with `visibility="public"`. Private rows are invisible to anonymous callers (404, not 401). Swagger UI: `/api/docs/`. OpenAPI: `/api/schema/`.
 
 ### Core flow
 
@@ -140,9 +142,11 @@ All endpoints live under `/api/v1/` and require `Authorization: Bearer <token>`.
 | POST | `/aois/` | Create an AOI polygon (GeoJSON Feature) |
 | GET | `/aois/` | List AOIs (bbox-filterable) |
 | POST | `/datasets/build/` | Enqueue a dataset build job. `aoi_ids`, `source_imagery` (TMS), `zoom`, `label_tasks`, `label_classes`, `keywords` (allowed: `building`, `road`, `tree`, `water`, `landuse`), `label_type`, `geometry_type` |
-| GET | `/datasets/{id}/?expand=stac` | Inspect dataset, with STAC metadata + presigned `chips`/`labels` URLs once published |
-| GET | `/local-models/` | List published local models (the *family*; STAC has per-version detail) |
+| GET | `/datasets/{id}/?expand=stac` | Inspect dataset, with STAC metadata + presigned `chips`/`labels` URLs once `status=built` |
+| POST | `/datasets/{id}/{publish,unpublish}/` | Toggle dataset `visibility` (anonymous read) |
+| GET | `/local-models/` | List local models (filterable by `status`, `visibility`, `user`) |
 | GET | `/local-models/{id}/runs/` | List ZenML pipeline runs that produced this model |
+| POST | `/local-models/{id}/{publish,unpublish}/` | Toggle local-model `visibility` (anonymous read) |
 | POST | `/trainings/submit/` | Enqueue a finetune. `base_model_stac_id`, `dataset_stac_id`, `model_name`, `overrides` (must match base-model's `fair:hyperparameters_spec`) |
 | GET | `/trainings/{id}/` | Run state including `zenml_run_id` |
 | GET | `/trainings/runs/{run_id}/status/` | Force-poll ZenML; refreshes the DB row |
@@ -153,7 +157,7 @@ All endpoints live under `/api/v1/` and require `Authorization: Bearer <token>`.
 | GET | `/predictions/{id}/` | Status + assets (presigned) once `results_ready=true` |
 | GET | `/predictions/{id}/result/` | Just the three presigned URLs (geojson / fgb / pmtiles); 409 until `results_ready` |
 | GET | `/predictions/runs/{run_id}/{status,logs}/` | Same shape as trainings |
-| POST | `/predictions/{id}/{publish,unpublish}/` | Toggle `is_public` (anonymous read of result) |
+| POST | `/predictions/{id}/{publish,unpublish}/` | Toggle prediction `visibility` (anonymous read of result) |
 
 ### Schema endpoints
 
@@ -203,11 +207,11 @@ Response carries `properties.id` -> call this `AOI_ID`.
 }
 ```
 
-Response: `id` (-> `DATASET_ID`), `stac_id` (-> `STAC_ID`), `build_status: "building"`.
+Response: `id` (-> `DATASET_ID`), `stac_id` (-> `STAC_ID`), `status: "building"`, `visibility: "private"`.
 
 **3. Wait for the build** : `GET /api/v1/datasets/{DATASET_ID}/`
 
-Poll until `build_status == "published"` (~2 min for this AOI).
+Poll until `status == "built"` (~2 min for this AOI). To make it readable by anonymous users, follow up with `POST /api/v1/datasets/{DATASET_ID}/publish/`.
 
 **4. Submit the fine-tune** : `POST /api/v1/trainings/submit/`
 

backend/README.md

@@ -42,6 +42,8 @@ OpenAPI schema at `/api/schema/`, Swagger UI at `/api/docs/`, ReDoc at `/api/red
 
 `AUTH_PROVIDER` selects the auth backend. Both share one contract: `Authorization: Bearer <token>`. `hanko` (production) validates a per-user JWT issued by Hanko (sent via Bearer header or `hanko` cookie). `dev` (local only) compares the Bearer token against the static `FAIR_DEV_TOKEN`; anyone with the token gets full dev-user access. Same header in dev and prod, only the issuer differs.
 
+`GET` on datasets, local-models, and predictions is open to anonymous callers for rows with `visibility="public"`. Owner-scoped lifecycle data (AOIs, trainings, feedback, notifications) and every write require Bearer auth.
+
 | Name | Required | Default | Description |
 |------|----------|---------|-------------|
 | `AUTH_PROVIDER` | no | `hanko` | One of `hanko`, `dev`. |

backend/accounts/permissions.py

@@ -30,3 +30,12 @@ def has_object_permission(self, request, view, obj) -> bool:
         if _is_admin(request.user):
             return True
         return getattr(obj, "user", None) == request.user
+
+
+class PublishedReadOrAuthenticatedWrite(permissions.BasePermission):
+    """View-level: SAFE methods open; non-SAFE requires authentication."""
+
+    def has_permission(self, request, view) -> bool:
+        if request.method in permissions.SAFE_METHODS:
+            return True
+        return bool(request.user and request.user.is_authenticated)

backend/datasets/admin.py

@@ -5,8 +5,8 @@
 
 @geoadmin.register(Dataset)
 class DatasetAdmin(geoadmin.GISModelAdmin):
-    list_display = ["title", "stac_id", "build_status", "user", "created_at"]
-    list_filter = ["build_status", "created_at"]
+    list_display = ["title", "stac_id", "status", "visibility", "user", "created_at"]
+    list_filter = ["status", "visibility", "created_at"]
     search_fields = ["title", "stac_id", "user__username"]
     readonly_fields = ["created_at", "last_modified"]
     date_hierarchy = "created_at"

backend/datasets/migrations/0003_remove_dataset_datasets_da_build_s_7fd7fa_idx_and_more.py

@@ -0,0 +1,77 @@
+from django.conf import settings
+from django.db import migrations, models
+
+
+def forward(apps, schema_editor):
+    Dataset = apps.get_model("datasets", "Dataset")
+    status_map = {
+        "draft": "draft",
+        "building": "building",
+        "published": "built",
+        "failed": "failed",
+    }
+    for row in Dataset.objects.all().only("id", "build_status"):
+        new_status = status_map.get(row.build_status, "draft")
+        new_visibility = "public" if row.build_status == "published" else "private"
+        Dataset.objects.filter(pk=row.pk).update(
+            status=new_status, visibility=new_visibility
+        )
+
+
+def backward(apps, schema_editor):
+    Dataset = apps.get_model("datasets", "Dataset")
+    status_map = {
+        "draft": "draft",
+        "building": "building",
+        "built": "published",
+        "failed": "failed",
+    }
+    for row in Dataset.objects.all().only("id", "status"):
+        Dataset.objects.filter(pk=row.pk).update(build_status=status_map.get(row.status, "draft"))
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("datasets", "0002_remove_dataset_build_error"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name="dataset",
+            name="datasets_da_build_s_7fd7fa_idx",
+        ),
+        migrations.AddField(
+            model_name="dataset",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("draft", "Draft"),
+                    ("building", "Building"),
+                    ("built", "Built"),
+                    ("failed", "Failed"),
+                ],
+                default="draft",
+                max_length=20,
+            ),
+        ),
+        migrations.AddField(
+            model_name="dataset",
+            name="visibility",
+            field=models.CharField(
+                choices=[("private", "Private"), ("public", "Public")],
+                db_index=True,
+                default="private",
+                max_length=20,
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="dataset",
+            index=models.Index(fields=["status"], name="datasets_da_status_f1863c_idx"),
+        ),
+        migrations.RunPython(forward, backward),
+        migrations.RemoveField(
+            model_name="dataset",
+            name="build_status",
+        ),
+    ]

backend/datasets/models.py

@@ -2,21 +2,23 @@
 from django.db import models, transaction
 
 from accounts.models import OsmUser
+from shared.enums import Visibility
 from shared.validators import validate_geometry
 
 
 class Dataset(models.Model):
-    class BuildStatus(models.TextChoices):
+    class Status(models.TextChoices):
         DRAFT = "draft", "Draft"
         BUILDING = "building", "Building"
-        PUBLISHED = "published", "Published"
+        BUILT = "built", "Built"
         FAILED = "failed", "Failed"
 
     stac_id = models.CharField(max_length=200, unique=True)
     title = models.CharField(max_length=200)
     source_imagery = models.URLField()
-    build_status = models.CharField(
-        max_length=20, choices=BuildStatus.choices, default=BuildStatus.DRAFT
+    status = models.CharField(max_length=20, choices=Status.choices, default=Status.DRAFT)
+    visibility = models.CharField(
+        max_length=20, choices=Visibility.choices, default=Visibility.PRIVATE, db_index=True
     )
     user = models.ForeignKey(
         OsmUser, to_field="osm_id", on_delete=models.CASCADE, related_name="datasets"
@@ -26,7 +28,7 @@ class BuildStatus(models.TextChoices):
 
     class Meta:
         indexes = [
-            models.Index(fields=["build_status"]),
+            models.Index(fields=["status"]),
             models.Index(fields=["user"]),
         ]
         ordering = ["-created_at"]

backend/datasets/serializers.py

@@ -96,7 +96,8 @@ class Meta:
             "stac_id",
             "title",
             "source_imagery",
-            "build_status",
+            "status",
+            "visibility",
             "stac_url",
             "user",
             "star_count",
@@ -109,7 +110,8 @@ class Meta:
         read_only_fields = [
             "id",
             "stac_id",
-            "build_status",
+            "status",
+            "visibility",
             "stac_url",
             "user",
             "star_count",
@@ -130,7 +132,7 @@ def get_stac(self, obj: Dataset) -> dict | None:
 
     @extend_schema_field(DatasetAssetsSerializer(allow_null=True))
     def get_assets(self, obj: Dataset) -> dict[str, str] | None:
-        if obj.build_status != Dataset.BuildStatus.PUBLISHED:
+        if obj.status != Dataset.Status.BUILT:
             return None
         from shared.storage import StoragePaths, presigned_get_url
 

backend/datasets/tasks.py

@@ -68,13 +68,13 @@ def build_dataset(
         client = for_user(str(dataset.user.osm_id))
         published_id = client.register_dataset(params, paths=BackendDatasetPaths)
         dataset.stac_id = published_id
-        dataset.build_status = Dataset.BuildStatus.PUBLISHED
-        dataset.save(update_fields=["stac_id", "build_status", "last_modified"])
+        dataset.status = Dataset.Status.BUILT
+        dataset.save(update_fields=["stac_id", "status", "last_modified"])
         invalidate_stac_cache(DATASETS_COLLECTION, published_id)
     except Exception:
         logger.exception("dataset build failed for %s", dataset_id)
-        dataset.build_status = Dataset.BuildStatus.FAILED
-        dataset.save(update_fields=["build_status", "last_modified"])
+        dataset.status = Dataset.Status.FAILED
+        dataset.save(update_fields=["status", "last_modified"])
         raise
 
 

backend/datasets/views.py

@@ -1,5 +1,6 @@
 import secrets
 
+from django.db.models import Q
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404
 from django.utils.text import slugify
@@ -15,7 +16,14 @@
 from rest_framework_gis.filters import InBBoxFilter
 
 from accounts.authentication import OsmAuthentication
-from accounts.permissions import IsAdmin, IsOwnerOrAdminOrReadOnly
+from accounts.permissions import (
+    IsAdmin,
+    IsOwnerOrAdmin,
+    IsOwnerOrAdminOrReadOnly,
+    PublishedReadOrAuthenticatedWrite,
+    _is_admin,
+)
+from shared.enums import Visibility
 from shared.integrations.stac import (
     DATASETS_COLLECTION,
     FAIR_PINNED_PROPERTY,
@@ -61,20 +69,28 @@ class DatasetViewSet(viewsets.ModelViewSet):
     queryset = Dataset.objects.all()
     serializer_class = DatasetSerializer
     authentication_classes = [OsmAuthentication]
-    permission_classes = [IsAuthenticated, IsOwnerOrAdminOrReadOnly]
+    permission_classes = [PublishedReadOrAuthenticatedWrite, IsOwnerOrAdminOrReadOnly]
     filter_backends = [DjangoFilterBackend, filters.OrderingFilter, filters.SearchFilter]
-    filterset_fields = ["build_status", "user"]
+    filterset_fields = ["status", "visibility", "user"]
     search_fields = ["title", "stac_id"]
     ordering_fields = ["created_at", "last_modified"]
 
     def get_queryset(self):
         from shared.stars import annotate_stars
 
-        return annotate_stars(Dataset.objects.all(), self.request)
+        qs = Dataset.objects.all()
+        user = self.request.user
+        if not user.is_authenticated:
+            qs = qs.filter(visibility=Visibility.PUBLIC)
+        elif not _is_admin(user):
+            qs = qs.filter(Q(user=user) | Q(visibility=Visibility.PUBLIC))
+        return annotate_stars(qs, self.request)
 
     def get_permissions(self):
         if self.action == "pin":
             return [IsAuthenticated(), IsAdmin()]
+        if self.action in {"publish", "unpublish", "build"}:
+            return [IsAuthenticated(), IsOwnerOrAdmin()]
         return super().get_permissions()
 
     def get_serializer_context(self):
@@ -107,6 +123,22 @@ def list(self, request, *args, **kwargs):
                 return self.get_paginated_response(serializer.data)
         return super().list(request, *args, **kwargs)
 
+    @extend_schema(request=None, responses={200: DatasetSerializer})
+    @action(detail=True, methods=["post"], url_path="publish")
+    def publish(self, request, pk: int | None = None) -> Response:
+        dataset = self.get_object()
+        dataset.visibility = Visibility.PUBLIC
+        dataset.save(update_fields=["visibility", "last_modified"])
+        return Response(self.get_serializer(dataset).data, status=status.HTTP_200_OK)
+
+    @extend_schema(request=None, responses={200: DatasetSerializer})
+    @action(detail=True, methods=["post"], url_path="unpublish")
+    def unpublish(self, request, pk: int | None = None) -> Response:
+        dataset = self.get_object()
+        dataset.visibility = Visibility.PRIVATE
+        dataset.save(update_fields=["visibility", "last_modified"])
+        return Response(self.get_serializer(dataset).data, status=status.HTTP_200_OK)
+
     @action(detail=True, methods=["patch"], url_path="pin")
     def pin(self, request, pk: int | None = None) -> Response:
         dataset = self.get_object()
@@ -141,7 +173,7 @@ def build(self, request) -> Response:
             stac_id=_slugify(payload["title"]),
             title=payload["title"],
             source_imagery=payload["source_imagery"],
-            build_status=Dataset.BuildStatus.BUILDING,
+            status=Dataset.Status.BUILDING,
             user=request.user,
         )
         aoi_qs.update(dataset=dataset)