Skip to content

Commit e3950aa

Browse files
stephenkilbournstephenkilbourn
authored
Edit stac collections (#135)
* set up navigation for future edit-existing-collections * remove legacy auth overrides now done in auth.ts * add stac:collection:update scope * add existing collections page
1 parent 715d6b2 commit e3950aa

File tree

62 files changed

+6267
-228
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+6267
-228
lines changed

‎.env.example‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,4 +17,7 @@ NEXT_PUBLIC_KEYCLOAK_ISSUER=
1717
NEXT_PUBLIC_MOCK_TENANTS=
1818
# Disable authentication entirely (uses mock user + mock tenants)
1919
# NEXT_PUBLIC_DISABLE_AUTH=true
20+
# Set to 'true' to show Edit Existing Collection options and allow API access
21+
ENABLE_EXISTING_COLLECTION_EDIT=true
22+
NEXT_PUBLIC_ENABLE_EXISTING_COLLECTION_EDIT=true
2023
NEXT_PUBLIC_ADDITIONAL_LOGO=

‎FEATURE_FLAG_CLEANUP.md‎

Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
# Edit Existing Collection Feature Flag Cleanup Guide
2+
3+
This document outlines all the changes needed to remove the `ENABLE_EXISTING_COLLECTION_EDIT` feature flag and make the Edit Existing Collection functionality permanently available.
4+
5+
## Overview
6+
7+
The feature flag was implemented to control visibility and access to the "Edit Existing Collection" feature across:
8+
9+
- UI components (MenuBar, CollectionsClient)
10+
- API endpoints (/api/existing-collection/\*)
11+
- Test configurations
12+
13+
## Files to Update
14+
15+
### 1. Environment Configuration
16+
17+
#### `.env.example`
18+
19+
**Remove these lines:**
20+
21+
```env
22+
# Enable or disable the Edit Existing Collection feature
23+
# Set to 'true' to show Edit Existing Collection options and allow API access
24+
ENABLE_EXISTING_COLLECTION_EDIT=true
25+
NEXT_PUBLIC_ENABLE_EXISTING_COLLECTION_EDIT=true
26+
```
27+
28+
### 2. Test Configuration Files
29+
30+
#### `playwright.config.ts`
31+
32+
**Remove** `ENABLE_EXISTING_COLLECTION_EDIT=true` from the command:
33+
34+
```typescript
35+
// FROM:
36+
command: 'NEXT_PUBLIC_DISABLE_AUTH=true NEXT_PUBLIC_MOCK_SCOPES="dataset:update stac:collection:update dataset:create" ENABLE_EXISTING_COLLECTION_EDIT=true yarn dev',
37+
38+
// TO:
39+
command: 'NEXT_PUBLIC_DISABLE_AUTH=true NEXT_PUBLIC_MOCK_SCOPES="dataset:update stac:collection:update dataset:create" yarn dev',
40+
```
41+
42+
#### `vitest.config.mts`
43+
44+
**Remove the env configuration:**
45+
46+
```typescript
47+
// REMOVE this entire env block:
48+
env: {
49+
ENABLE_EXISTING_COLLECTION_EDIT: 'true',
50+
NEXT_PUBLIC_ENABLE_EXISTING_COLLECTION_EDIT: 'true',
51+
},
52+
```
53+
54+
### 3. UI Components
55+
56+
#### `components/layout/MenuBar.tsx`
57+
58+
**Remove the environment variable check:**
59+
60+
```typescript
61+
// REMOVE this line:
62+
const isEditExistingCollectionEnabled =
63+
process.env.NEXT_PUBLIC_ENABLE_EXISTING_COLLECTION_EDIT === 'true';
64+
```
65+
66+
**Simplify the menu items array:**
67+
68+
```typescript
69+
// FROM conditional spread syntax:
70+
...(isEditExistingCollectionEnabled ? [{
71+
key: '/edit-existing-collection',
72+
// ... menu item definition
73+
}] : []),
74+
75+
// TO permanent menu item:
76+
{
77+
key: '/edit-existing-collection',
78+
label:
79+
hasLimitedAccess || !hasEditStacCollectionPermission ? (
80+
<Tooltip
81+
title="Contact the VEDA Data Services team for access"
82+
placement="right"
83+
>
84+
<span style={{ cursor: 'not-allowed' }}>
85+
<Link href="/edit-existing-collection">
86+
Edit Existing Collection
87+
</Link>
88+
</span>
89+
</Tooltip>
90+
) : (
91+
<Link href="/edit-existing-collection">
92+
Edit Existing Collection
93+
</Link>
94+
),
95+
icon: <DatabaseOutlined />,
96+
disabled: hasLimitedAccess || !hasEditStacCollectionPermission,
97+
},
98+
```
99+
100+
#### `app/collections/_components/CollectionsClient.tsx`
101+
102+
**Remove the environment variable check:**
103+
104+
```typescript
105+
// REMOVE this line:
106+
const isEditExistingCollectionEnabled =
107+
process.env.NEXT_PUBLIC_ENABLE_EXISTING_COLLECTION_EDIT === 'true';
108+
```
109+
110+
**Remove conditional rendering - make sections permanent:**
111+
112+
For **Limited Access View**:
113+
114+
```typescript
115+
// FROM:
116+
{isEditExistingCollectionEnabled && (
117+
<>
118+
<Title level={3} style={{ marginTop: 40 }}>
119+
Existing STAC Collections
120+
</Title>
121+
<Row gutter={16} style={{ marginTop: 16 }}>
122+
{/* ... card content */}
123+
</Row>
124+
</>
125+
)}
126+
127+
// TO:
128+
<>
129+
<Title level={3} style={{ marginTop: 40 }}>
130+
Existing STAC Collections
131+
</Title>
132+
<Row gutter={16} style={{ marginTop: 16 }}>
133+
{/* ... card content */}
134+
</Row>
135+
</>
136+
```
137+
138+
For **Main View**:
139+
140+
```typescript
141+
// FROM:
142+
{isEditExistingCollectionEnabled && (
143+
<>
144+
<Title level={3} style={{ marginTop: 40 }}>
145+
Existing STAC Collections
146+
</Title>
147+
<Row gutter={16} style={{ marginTop: 16 }}>
148+
{/* ... card content */}
149+
</Row>
150+
</>
151+
)}
152+
153+
// TO:
154+
<>
155+
<Title level={3} style={{ marginTop: 40 }}>
156+
Existing STAC Collections
157+
</Title>
158+
<Row gutter={16} style={{ marginTop: 16 }}>
159+
{/* ... card content */}
160+
</Row>
161+
</>
162+
```
163+
164+
### 4. API Endpoints
165+
166+
#### `app/api/existing-collection/route.ts`
167+
168+
**Remove the feature flag check:**
169+
170+
```typescript
171+
// REMOVE this entire block:
172+
// Check if the Edit Existing Collection feature is enabled
173+
if (process.env.ENABLE_EXISTING_COLLECTION_EDIT !== 'true') {
174+
return NextResponse.json(
175+
{ error: 'Edit Existing Collection feature is disabled' },
176+
{ status: 403 }
177+
);
178+
}
179+
```
180+
181+
#### `app/api/existing-collection/[collectionId]/route.ts`
182+
183+
**Remove the feature flag check from both GET and PUT methods:**
184+
185+
```typescript
186+
// REMOVE this entire block from both functions:
187+
// Check if the Edit Existing Collection feature is enabled
188+
if (process.env.ENABLE_EXISTING_COLLECTION_EDIT !== 'true') {
189+
return NextResponse.json(
190+
{ error: 'Edit Existing Collection feature is disabled' },
191+
{ status: 403 }
192+
);
193+
}
194+
```
195+
196+
## Cleanup Steps
197+
198+
1. **Update environment files** - Remove feature flag variables
199+
2. **Update test configurations** - Remove env vars from test configs
200+
3. **Update UI components** - Remove conditional rendering logic
201+
4. **Update API endpoints** - Remove feature flag checks
202+
5. **Test thoroughly** - Ensure Edit Existing Collection works in all scenarios
203+
6. **Update documentation** - Remove references to the feature flag
204+
7. **Clean up any remaining environment variables** in production/staging configs
205+
206+
## Verification Checklist
207+
208+
After cleanup, verify:
209+
210+
- [ ] Edit Existing Collection appears in MenuBar for users with `stac:collection:update` scope
211+
- [ ] Edit Existing Collection section shows in Collections page when feature flag env vars are absent
212+
- [ ] API endpoints `/api/existing-collection/*` work without feature flag
213+
- [ ] Unit tests pass without feature flag environment variables
214+
- [ ] Playwright tests work without feature flag in command
215+
- [ ] No console errors about missing environment variables
216+
217+
## Notes
218+
219+
- The feature flag was a **client-side and server-side** implementation
220+
- **Client-side**: `NEXT_PUBLIC_ENABLE_EXISTING_COLLECTION_EDIT` (visible in browser)
221+
- **Server-side**: `ENABLE_EXISTING_COLLECTION_EDIT` (API routes only)
222+
- Both must be removed for complete cleanup
223+
- Consider doing this cleanup in a dedicated PR for easier review and rollback if needed

‎FormSchemas/collections/collectionSchema.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,8 +87,7 @@
8787
"maxItems": 2,
8888
"items": {
8989
"type": ["string", "null"],
90-
"format": "date-time",
91-
"pattern": "(\\+00:00|Z)$"
90+
"pattern": "(\\+00:00|\\+00|Z)$"
9291
}
9392
}
9493
}

‎README.md‎

Lines changed: 53 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,13 @@ https://nasa-impact.github.io/veda-ingest-ui/
3434

3535
```
3636
├── app/ # Next.js App Router pages
37-
│ ├── api/ # API routes for GitHub operations
37+
│ ├── api/ # API routes for GitHub operations and STAC API
3838
│ ├── collections/ # Collection management pages
3939
│ ├── datasets/ # Dataset management pages
40+
│ ├── edit-existing-collection/ # STAC collection editing interface
4041
│ └── upload/ # File upload functionality
4142
├── components/ # Reusable React components
43+
│ ├── ingestion/ # Form components for data ingestion and editing
4244
│ ├── layout/ # Layout components (header, sidebar, etc.)
4345
│ ├── rjsf-components/ # Custom RJSF form components
4446
│ ├── thumbnails/ # Thumbnail upload components
@@ -53,17 +55,35 @@ https://nasa-impact.github.io/veda-ingest-ui/
5355

5456
# Architecture
5557

56-
The application is designed to allow users to create and edit PRs in the veda-data repository.New PRs are created with a prefix of `'Ingest Request for [collectionName]'`. The branch name and file name of the json for these new PRs is set by the Collection Name field in the form after any non-alphanumeric characters are removed from the collection name:
58+
The application supports two primary workflows:
59+
60+
## 1. Data Ingestion
61+
62+
The application allows users to create and edit PRs in the veda-data repository for data ingestion. New PRs are created with a prefix of `'[collection/dataset] Ingest Request for [collectionName]'`. The branch name and file name of the json for these new PRs is set by the Collection Name field in the form after any non-alphanumeric characters are removed from the collection name:
5763

5864
```
59-
const fileName = 'ingestion-data/staging/dataset-config/${collectionName}.json`;
65+
const fileName = 'ingestion-data/staging/dataset-config/${collectionName}.json';
6066
const branchName = `feat/${collectionName}`;
6167
```
6268

63-
All API calls require users to be authenticated via keycloak. The API then obtains a github token and makes the desired calls with the github token.
64-
6569
Users are allowed to edit open PRs that are modifying json files in the standard filepath for each ingestion type. The existing values in the json will be loaded into a form. A user can update those values and a new commit will be added to the PR with the new values.
6670

71+
## 2. Collection Editing
72+
73+
The application also provides direct editing of existing STAC collections through the STAC API. User must have `stac:collection:update` scope for editing permissions.
74+
75+
- **Collection Discovery**: Browse existing collections from `https://staging.openveda.cloud/api/stac/collections`
76+
- **Real-time Editing**: Modify collection metadata directly without GitHub PRs
77+
- **Data Sanitization**: Automatic STAC schema compliance with null-to-array/object conversion and datetime format fixes
78+
79+
## Authentication & Authorization
80+
81+
All API calls require users to be authenticated via Keycloak.
82+
83+
- **GitHub Operations**: Uses GitHub token for repository operations
84+
- **STAC Operations**: Uses access token for STAC API calls with tenant-based permissions
85+
- **Tenant Filtering**: Support for multi-tenant environments with proper access controls
86+
6787
## Creation Component Architecture
6888

6989
```mermaid
@@ -196,11 +216,37 @@ yarn dev
196216

197217
This will start the app and make it available at <http://localhost:3000/>.
198218

199-
To bypass the keycloak login, set the `NEXT_PUBLIC_DISABLE_AUTH` environment variable to true. This variable is als leveraged for Playwright testing.
219+
To bypass the keycloak login, set the `NEXT_PUBLIC_DISABLE_AUTH` environment variable to true. This variable is also leveraged for Playwright testing.
220+
221+
## 🛠️ STAC Data Sanitization
222+
223+
To fix incorrect, previously ingested data, the application includes a data sanitization system to ensure STAC schema compliance:
224+
225+
### Sanitization Features
226+
227+
- **Null Handling**: Converts `null` values to appropriate empty arrays or objects
228+
- **Datetime Format**: Fixes timezone and separator issues (e.g., `+00` → `+00:00`, space → `T`)
229+
230+
### Implementation
231+
232+
Sanitization logic is located in `utils/stacSanitization.ts` and includes:
233+
234+
```typescript
235+
// Main sanitization function
236+
import { sanitizeFormData } from '@/utils/stacSanitization';
237+
238+
const cleanedData = sanitizeFormData(formData);
239+
```
240+
241+
### Field Type Rules
242+
243+
- **Arrays**: `stac_extensions`, `keywords`, `providers`, `links`
244+
- **Objects**: `assets`, `item_assets`, `summaries`
245+
- **Datetime Strings**: Temporal extent fields with format fixes
200246

201247
## Configuring the Validation Form
202248

203-
The fields in the Validation Form are configured by a combination of the json schema in the [jsonschema.json file](FormSchemas/jsonschema.json) and the UI Schema in the [uischema.json file](FormSchemas/uischema.json). To modify fields in the form, a developer must update the json schema to include the proper JSON schema data fields and then modify the ui Schema to have any new or renamed fields in the desired location.
249+
The fields in the Validation Form are configured by a combination of the json schema in the [jsonschema.json file](FormSchemas/**/jsonschema.json) and the UI Schema in the [uischema.json file](FormSchemas/**/uischema.json). To modify fields in the form, a developer must update the json schema to include the proper JSON schema data fields and then modify the ui Schema to have any new or renamed fields in the desired location.
204250

205251
The Form uses a 24 column grid format and the layout of each row is dictated by the "ui:grid" array in that json. Each row is defined as an object with each field allowed up to 24 columns wide. For example:
206252

‎__mocks__/handlers.ts‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import { githubResponse } from './githubResponse';
44
import { retrieveIngestResponse } from './retrieveIngestResponse';
55
import { collectionIngestResponse } from './collectionIngestResponse';
66
import { extensionSchemaResponse } from './extensionSchemaResponse';
7+
import { stacCollectionsResponse } from './stacCollectionsResponse';
8+
import { stacCollectionResponse } from './stacCollectionResponse';
79
// --- Placeholder Tile Logic ---
810
const MOCK_TILE_BASE64 =
911
'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=';
@@ -34,7 +36,7 @@ const mockSession = {
3436
email: '[email protected]',
3537
image: null,
3638
},
37-
scopes: ['dataset:update'],
39+
scopes: ['dataset:update', 'stac:collection:update'],
3840
tenants: ['tenant1', 'tenant2', 'tenant3'],
3941
expires: new Date(Date.now() + 24 * 60 * 60 * 1000).toISOString(),
4042
};
@@ -224,4 +226,33 @@ export const handlers = [
224226
http.get('https://staging.openveda.cloud/api/raster/cog/validate', () => {
225227
return HttpResponse.json({ COG: true });
226228
}),
229+
230+
http.get('/api/existing-collection', ({ request }) => {
231+
const url = new URL(request.url);
232+
const tenant = url.searchParams.get('tenant');
233+
234+
let filteredResponse = { ...stacCollectionsResponse };
235+
236+
// Filter by tenant if specified
237+
if (tenant) {
238+
filteredResponse.collections = stacCollectionsResponse.collections.filter(
239+
(collection: any) => {
240+
if (tenant === 'Public') {
241+
return !collection.tenant || collection.tenant === '';
242+
}
243+
return collection.tenant === tenant;
244+
}
245+
);
246+
}
247+
248+
return HttpResponse.json(filteredResponse);
249+
}),
250+
251+
http.get('/api/existing-collection/:collectionId', ({ params }) => {
252+
// Return the same mock collection response for any collection ID
253+
return HttpResponse.json({
254+
...stacCollectionResponse,
255+
id: params.collectionId,
256+
});
257+
}),
227258
];

0 commit comments

Comments
 (0)