diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index aacc4dcd3..4609eb697 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -80,6 +80,7 @@ jobs: POSTGRES_DB: postgres OSMCHA_API_KEY: ${{ secrets.OSMCHA_API_KEY }} DJANGO_SECRET_KEY: test-django-secret-key + MAPILLARY_API_KEY: ${{ secrets.MAPILLARY_API_KEY }} COMPOSE_FILE: ../docker-compose.yaml:../docker-compose-ci.yaml run: | docker compose run --rm mapswipe_workers_creation python -m unittest discover --verbose --start-directory tests/unittests/ diff --git a/django/apps/existing_database/models.py b/django/apps/existing_database/models.py index 0cf6582fb..5bc85e113 100644 --- a/django/apps/existing_database/models.py +++ b/django/apps/existing_database/models.py @@ -66,6 +66,9 @@ class Type(models.IntegerChoices): FOOTPRINT = 2, "Validate" CHANGE_DETECTION = 3, "Compare" COMPLETENESS = 4, "Completeness" + MEDIA = 5, "Media" + DIGITIZATION = 6, "Digitization" + STREET = 7, "Street" project_id = models.CharField(primary_key=True, max_length=999) created = models.DateTimeField(blank=True, null=True) @@ -127,7 +130,7 @@ class Task(Model): project = models.ForeignKey(Project, models.DO_NOTHING, related_name="+") group_id = models.CharField(max_length=999) task_id = models.CharField(max_length=999) - geom = gis_models.MultiPolygonField(blank=True, null=True) + geom = gis_models.GeometryField(blank=True, null=True) # Database uses JSON instead of JSONB (not supported by django) project_type_specifics = models.TextField(blank=True, null=True) diff --git a/django/schema.graphql b/django/schema.graphql index d0187f7f1..b5596fc46 100644 --- a/django/schema.graphql +++ b/django/schema.graphql @@ -97,6 +97,9 @@ enum ProjectTypeEnum { FOOTPRINT CHANGE_DETECTION COMPLETENESS + MEDIA + DIGITIZATION + STREET } type ProjectTypeSwipeStatsType { diff --git a/docker-compose.yaml b/docker-compose.yaml index ccbc2aa9d..d465a704c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -76,6 +76,7 @@ x-mapswipe-workers: &base_mapswipe_workers SLACK_CHANNEL: '${SLACK_CHANNEL}' SENTRY_DSN: '${SENTRY_DSN}' OSMCHA_API_KEY: '${OSMCHA_API_KEY}' + MAPILLARY_API_KEY: '${MAPILLARY_API_KEY}' depends_on: - postgres volumes: diff --git a/example.env b/example.env index f4e8c46a1..2fa92ae33 100644 --- a/example.env +++ b/example.env @@ -75,3 +75,6 @@ COMMUNITY_DASHBOARD_GRAPHQL_ENDPOINT=https://api.example.com/graphql/ COMMUNITY_DASHBOARD_SENTRY_DSN= COMMUNITY_DASHBOARD_SENTRY_TRACES_SAMPLE_RATE= COMMUNITY_DASHBOARD_MAPSWIPE_WEBSITE=https://mapswipe.org + +# Mapillary +MAPILLARY_API_KEY= \ No newline at end of file diff --git a/manager-dashboard/app/Base/configs/projectTypes.ts b/manager-dashboard/app/Base/configs/projectTypes.ts index fd8d2df8e..e2f7f74eb 100644 --- a/manager-dashboard/app/Base/configs/projectTypes.ts +++ b/manager-dashboard/app/Base/configs/projectTypes.ts @@ -3,6 +3,7 @@ import { PROJECT_TYPE_BUILD_AREA, PROJECT_TYPE_FOOTPRINT, PROJECT_TYPE_CHANGE_DETECTION, + PROJECT_TYPE_STREET, PROJECT_TYPE_COMPLETENESS, } from '#utils/common'; @@ -15,6 +16,7 @@ const mapswipeProjectTypeOptions: { { value: PROJECT_TYPE_BUILD_AREA, label: 'Find' }, { value: PROJECT_TYPE_FOOTPRINT, label: 'Validate' }, { value: PROJECT_TYPE_CHANGE_DETECTION, label: 'Compare' }, + { value: PROJECT_TYPE_STREET, label: 'Street' }, { value: PROJECT_TYPE_COMPLETENESS, label: 'Completeness' }, ]; diff --git a/manager-dashboard/app/Base/styles.css b/manager-dashboard/app/Base/styles.css index 306f01caf..c746dc570 100644 --- a/manager-dashboard/app/Base/styles.css +++ b/manager-dashboard/app/Base/styles.css @@ -122,9 +122,14 @@ p { --line-height-relaxed: 1.625; --line-height-loose: 2; - --shadow-card: 0 2px 4px -2px var(--color-shadow); --duration-transition-medium: .2s; + --color-background-hover-light: rgba(0, 0, 0, .04); + --width-calendar-date: 2.4rem; + + --opacity-watermark: 0.3; + --color-text-disabled: rgba(0, 0, 0, .3); + } diff --git a/manager-dashboard/app/components/Calendar/CalendarDate/index.tsx b/manager-dashboard/app/components/Calendar/CalendarDate/index.tsx new file mode 100644 index 000000000..2b1fed1a0 --- /dev/null +++ b/manager-dashboard/app/components/Calendar/CalendarDate/index.tsx @@ -0,0 +1,68 @@ +import React from 'react'; +import { _cs } from '@togglecorp/fujs'; + +import RawButton, { Props as RawButtonProps } from '../../RawButton'; +import { ymdToDateString, typedMemo } from '../../../utils/common.tsx'; + +import styles from './styles.css'; + +export interface Props { + className?: string; + year: number; + month: number; + date: number; + currentYear: number; + currentMonth: number; + activeDate?: string; + currentDate: number; + onClick?: (year: number, month: number, date: number) => void; + elementRef?: RawButtonProps['elementRef']; + ghost?: boolean; +} + +function CalendarDate(props: Props) { + const { + className, + year, + month, + date, + currentYear, + currentMonth, + currentDate, + onClick, + elementRef, + activeDate, + ghost, + } = props; + + const handleClick = React.useCallback(() => { + if (onClick) { + onClick(year, month, date); + } + }, [year, month, date, onClick]); + + const dateString = ymdToDateString(year, month, date); + + return ( + + {date} + + + ); +} + +export default typedMemo(CalendarDate); diff --git a/manager-dashboard/app/components/Calendar/CalendarDate/styles.css b/manager-dashboard/app/components/Calendar/CalendarDate/styles.css new file mode 100644 index 000000000..cf87157ab --- /dev/null +++ b/manager-dashboard/app/components/Calendar/CalendarDate/styles.css @@ -0,0 +1,25 @@ +.date { + border-radius: 50%; + width: var(--width-calendar-date); + height: var(--width-calendar-date); + + &.today { + color: var(--color-accent); + font-weight: var(--font-weight-bold); + } + + &:hover { + background-color: var(--color-background-hover-light); + } + + &.active { + background-color: var(--color-accent); + color: var(--color-text-on-dark); + pointer-events: none; + } + + &.ghost { + opacity: 0.5; + } +} + diff --git a/manager-dashboard/app/components/Calendar/index.tsx b/manager-dashboard/app/components/Calendar/index.tsx new file mode 100644 index 000000000..d72f9054d --- /dev/null +++ b/manager-dashboard/app/components/Calendar/index.tsx @@ -0,0 +1,292 @@ +import React from 'react'; +import { + _cs, + isNotDefined, + isDefined, +} from '@togglecorp/fujs'; +import { + IoTimeOutline, + IoChevronForward, + IoChevronBack, + IoCalendarOutline, +} from 'react-icons/io5'; + +import Button from '../Button'; +import NumberInput from '../NumberInput'; +import SelectInput from '../SelectInput'; +import useInputState from '../../hooks/useInputState'; +import { typedMemo } from '../../utils/common.tsx'; + +import CalendarDate, { Props as CalendarDateProps } from './CalendarDate'; + +import styles from './styles.css'; + +const weekDayNames = [ + 'Sunday', + 'Monday', + 'Tuesday', + 'Wednesday', + 'Thursday', + 'Friday', + 'Saturday', +]; + +interface MonthName { + key: number; + label: string; +} + +const monthNameList: MonthName[] = [ + { key: 0, label: 'January' }, + { key: 1, label: 'February' }, + { key: 2, label: 'March' }, + { key: 3, label: 'April' }, + { key: 4, label: 'May' }, + { key: 5, label: 'June' }, + { key: 6, label: 'July' }, + { key: 7, label: 'August' }, + { key: 8, label: 'September' }, + { key: 9, label: 'October' }, + { key: 10, label: 'November' }, + { key: 11, label: 'December' }, +]; + +function getStartOfWeek(year: number, month: number) { + return new Date(year, month, 1).getDay(); +} + +function getNumDaysInMonth(year: number, month: number) { + // Setting date to 0 will switch the date to last day of previous month + return new Date(year, month + 1, 0).getDate(); +} + +interface RenderDate { + type: 'prevMonth' | 'currentMonth' | 'nextMonth'; + date: number; +} + +function getDates(year: number, month: number) { + const numDays = getNumDaysInMonth(year, month); + const numDayInPrevMonth = getNumDaysInMonth(year, month - 1); + const startOfWeek = getStartOfWeek(year, month); + + const dates: RenderDate[] = []; + + for (let i = 0; i < startOfWeek; i += 1) { + dates.push({ + type: 'prevMonth', + date: numDayInPrevMonth - startOfWeek + i + 1, + }); + } + + for (let i = 0; i < numDays; i += 1) { + dates.push({ + type: 'currentMonth', + date: i + 1, + }); + } + + // 6 rows x 7 cols + const remainingDates = 42 - dates.length; + + for (let i = 0; i < remainingDates; i += 1) { + dates.push({ + type: 'nextMonth', + date: i + 1, + }); + } + + return dates; +} + +const monthKeySelector = (m: MonthName) => m.key; +const monthLabelSelector = (m: MonthName) => m.label; + +type RendererOmissions = 'year' | 'month' | 'date' | 'currentYear' | 'currentMonth' | 'currentDate' | 'onClick' | 'activeDate' | 'ghost'; +export interface Props

{ + className?: string; + dateRenderer?: (props: P) => React.ReactElement; + rendererParams?: (day: number, month: number, year: number) => Omit; + onDateClick?: (day: number, month: number, year: number) => void; + monthSelectionPopupClassName?: string; + initialDate?: string; + activeDate?: string; +} + +function Calendar

(props: Props

) { + const { + className, + dateRenderer: DateRenderer = CalendarDate, + rendererParams, + onDateClick, + monthSelectionPopupClassName, + initialDate, + activeDate, + } = props; + + const today = new Date(); + const current = initialDate ? new Date(initialDate) : today; + const currentYear = current.getFullYear(); + const currentMonth = current.getMonth(); + + const [year, setYear] = useInputState(currentYear); + const [month, setMonth] = useInputState(currentMonth); + + const dates = year ? getDates(year, month) : undefined; + + const handleGotoCurrentButtonClick = React.useCallback(() => { + const date = new Date(); + setYear(date.getFullYear()); + setMonth(date.getMonth()); + }, [setMonth, setYear]); + + const handleNextMonthButtonClick = React.useCallback(() => { + if (isDefined(year)) { + const date = new Date(year, month + 1, 1); + setYear(date.getFullYear()); + setMonth(date.getMonth()); + } + }, [year, month, setMonth, setYear]); + + const handlePreviousMonthButtonClick = React.useCallback(() => { + if (isDefined(year)) { + const date = new Date(year, month - 1, 1); + setYear(date.getFullYear()); + setMonth(date.getMonth()); + } + }, [year, month, setMonth, setYear]); + + const isValidYear = React.useMemo(() => { + if (isNotDefined(year)) { + return false; + } + + if (year < 1900 || year > 9999) { + return false; + } + + return true; + }, [year]); + + return ( +

+
+
+
+ +
+
+ +
+
+
+ {weekDayNames.map((wd) => ( +
+ {wd.substr(0, 2)} +
+ ))} +
+
+ {(isValidYear && isDefined(year) && dates) ? ( +
+ {dates.map((date) => { + let newMonth = month; + if (date.type === 'prevMonth') { + newMonth -= 1; + } else if (date.type === 'nextMonth') { + newMonth += 1; + } + const ymd = new Date(year, newMonth, date.date); + + const defaultProps: Pick = { + onClick: onDateClick, + year: ymd.getFullYear(), + month: ymd.getMonth(), + date: ymd.getDate(), + currentYear: today.getFullYear(), + currentMonth: today.getMonth(), + currentDate: today.getDate(), + activeDate, + ghost: date.type === 'prevMonth' || date.type === 'nextMonth', + }; + + const combinedProps = { + ...(rendererParams ? rendererParams( + date.date, month, year, + ) : undefined), + ...defaultProps, + } as P; + + const children = ( + + ); + + return ( +
+ {children} +
+ ); + })} +
+ ) : ( +
+ + Please select a valid year and month to view the dates +
+ )} +
+ + + +
+
+ ); +} + +export default typedMemo(Calendar); diff --git a/manager-dashboard/app/components/Calendar/styles.css b/manager-dashboard/app/components/Calendar/styles.css new file mode 100644 index 000000000..8d31cba85 --- /dev/null +++ b/manager-dashboard/app/components/Calendar/styles.css @@ -0,0 +1,93 @@ +.calendar { + display: flex; + flex-direction: column; + + .header { + flex-shrink: 0; + + .info { + display: flex; + align-items: flex-end; + justify-content: center; + padding: calc(var(--spacing-medium) - var(--spacing-small)); + + .current-year { + flex-basis: 40%; + padding: var(--spacing-small); + font-size: var(--font-size-large); + } + + .current-month { + flex-basis: 60%; + padding: var(--spacing-small); + } + } + + .week-days { + display: flex; + padding: calc(var(--spacing-medium) - var(--spacing-small)); + + .week-day-name { + display: flex; + align-items: center; + flex-basis: calc(100% / 7); + flex-shrink: 0; + justify-content: center; + padding: var(--spacing-small); + font-weight: var(--font-weight-bold); + } + } + } + + .day-list { + display: flex; + flex-grow: 1; + flex-wrap: wrap; + padding: calc(var(--spacing-medium) - var(--spacing-small)) var(--spacing-medium); + + .day-container { + --width: calc(100% / 7); + display: flex; + align-items: center; + flex-basis: var(--width); + justify-content: center; + width: var(--width); + } + } + + .empty-day-list { + display: flex; + align-items: center; + flex-direction: column; + flex-grow: 1; + justify-content: center; + padding: var(--spacing-large); + text-align: center; + color: var(--color-text); + + .icon { + opacity: var(--opacity-watermark); + margin: var(--spacing-medium); + font-size: var(--font-size-ultra-large); + } + } + + .actions { + display: flex; + flex-shrink: 0; + justify-content: flex-end; + padding: calc(var(--spacing-medium) - var(--spacing-small)); + + >* { + margin: var(--spacing-small) calc(var(--spacing-medium) - var(--spacing-small)); + } + } +} + +.month-selection-popup { + min-width: 10rem; + + .popup-content { + width: 100%!important; + } +} diff --git a/manager-dashboard/app/components/DateRangeInput/index.tsx b/manager-dashboard/app/components/DateRangeInput/index.tsx new file mode 100644 index 000000000..6442fc835 --- /dev/null +++ b/manager-dashboard/app/components/DateRangeInput/index.tsx @@ -0,0 +1,420 @@ +import React, { useMemo } from 'react'; +import { + _cs, + randomString, + isDefined, + isNotDefined, +} from '@togglecorp/fujs'; +import { + IoCalendarOutline, + IoClose, +} from 'react-icons/io5'; + +import useBlurEffect from '../../hooks/useBlurEffect'; +import useBooleanState from '../../hooks/useBooleanState'; +import InputContainer, { Props as InputContainerProps } from '../InputContainer'; +import RawInput from '../RawInput'; +import RawButton from '../RawButton'; +import Button from '../Button'; +import Popup from '../Popup'; +import Calendar, { Props as CalendarProps } from '../Calendar'; +import CalendarDate, { Props as CalendarDateProps } from '../Calendar/CalendarDate'; +import { ymdToDateString, dateStringToDate } from '../../utils/common.tsx'; + +import { + predefinedDateRangeOptions, + PredefinedDateRangeKey, +} from './predefinedDateRange'; + +import styles from './styles.css'; + +// FIXME: this is problematic when on end months +function prevMonth(date: Date) { + const newDate = new Date(date); + newDate.setMonth(newDate.getMonth() - 1); + return newDate; +} +function sameMonth(foo: Date, bar: Date) { + return foo.getFullYear() === bar.getFullYear() && foo.getMonth() === bar.getMonth(); +} + +export interface Value { + startDate?: string; + endDate?: string; +} + +interface DateRendererProps extends CalendarDateProps { + startDate?: string; + endDate?: string; +} + +function DateRenderer(props: DateRendererProps) { + const { + className: dateClassName, + year, + month, + date, + startDate, + endDate, + ghost, + ...otherProps + } = props; + + const start = startDate ? dateStringToDate(startDate).getTime() : undefined; + const end = endDate ? dateStringToDate(endDate).getTime() : undefined; + const current = new Date(year, month, date).getTime(); + + const inBetween = isDefined(start) && isDefined(end) && current > start && current < end; + + const dateString = ymdToDateString(year, month, date); + + const isEndDate = dateString === endDate; + const isStartDate = dateString === startDate; + + return ( + + ); +} + +type NameType = string | number | undefined; + +type InheritedProps = Omit; +export interface Props extends InheritedProps { + inputElementRef?: React.RefObject; + inputClassName?: string; + value: Value | undefined | null; + name: N; + onChange?: (value: Value | undefined, name: N) => void; + placeholder?: string; +} + +function DateRangeInput(props: Props) { + const { + actions, + actionsContainerClassName, + className, + disabled, + error, + errorContainerClassName, + hint, + hintContainerClassName, + icons, + iconsContainerClassName, + inputSectionClassName, + label, + labelContainerClassName, + readOnly, + inputElementRef, + containerRef: containerRefFromProps, + inputSectionRef: inputSectionRefFromProps, + inputClassName, + onChange, + name, + value, + placeholder, + } = props; + + const [tempDate, setTempDate] = React.useState>({ + startDate: undefined, + endDate: undefined, + }); + const [calendarMonthSelectionPopupClassName] = React.useState(randomString(16)); + const createdContainerRef = React.useRef(null); + const createdInputSectionRef = React.useRef(null); + const popupRef = React.useRef(null); + + const containerRef = containerRefFromProps ?? createdContainerRef; + const inputSectionRef = inputSectionRefFromProps ?? createdInputSectionRef; + const [ + showCalendar, + setShowCalendarTrue, + setShowCalendarFalse,, + toggleShowCalendar, + ] = useBooleanState(false); + + const hideCalendar = React.useCallback(() => { + setTempDate({ + startDate: undefined, + endDate: undefined, + }); + setShowCalendarFalse(); + }, [setShowCalendarFalse]); + + const handlePopupBlur = React.useCallback( + (isClickedWithin: boolean, e: MouseEvent) => { + // Following is to prevent the popup blur when + // month selection is changed in the calendar + const container = document.getElementsByClassName( + calendarMonthSelectionPopupClassName, + )[0]; + const isContainerOrInsideContainer = container + ? container === e.target || container.contains(e.target as HTMLElement) + : false; + if (!isClickedWithin && !isContainerOrInsideContainer) { + hideCalendar(); + } + }, + [hideCalendar, calendarMonthSelectionPopupClassName], + ); + + useBlurEffect( + showCalendar, + handlePopupBlur, + popupRef, + inputSectionRef, + ); + + const dateRendererParams = React.useCallback(() => ({ + startDate: tempDate.startDate ?? value?.startDate, + // we only set end date if user hasn't set the start date + // i.e. to show previously selected end date) + endDate: !tempDate.startDate ? value?.endDate : undefined, + }), [tempDate.startDate, value]); + + const handleCalendarDateClick: CalendarProps['onDateClick'] = React.useCallback( + (year, month, day) => { + setTempDate((prevTempDate) => { + if (isDefined(prevTempDate.startDate)) { + const lastDate = ymdToDateString(year, month, day); + + const prev = dateStringToDate(prevTempDate.startDate).getTime(); + const current = new Date(year, month, day).getTime(); + + const startDate = prev > current ? lastDate : prevTempDate.startDate; + const endDate = prev > current ? prevTempDate.startDate : lastDate; + + return { + startDate, + endDate, + }; + } + + return { + startDate: ymdToDateString(year, month, day), + endDate: undefined, + }; + }); + }, + [], + ); + + React.useEffect(() => { + if (isDefined(tempDate.endDate)) { + if (onChange) { + onChange(tempDate as Value, name); + } + hideCalendar(); + } + }, [tempDate, hideCalendar, onChange, name]); + + const handlePredefinedOptionClick = React.useCallback((optionKey: PredefinedDateRangeKey) => { + if (onChange) { + const option = predefinedDateRangeOptions.find((d) => d.key === optionKey); + + if (option) { + const { + startDate, + endDate, + } = option.getValue(); + + onChange({ + startDate: ymdToDateString( + startDate.getFullYear(), + startDate.getMonth(), + startDate.getDate(), + ), + endDate: ymdToDateString( + endDate.getFullYear(), + endDate.getMonth(), + endDate.getDate(), + ), + }, name); + } + } + + hideCalendar(); + }, [onChange, hideCalendar, name]); + + const handleClearButtonClick = React.useCallback(() => { + if (onChange) { + onChange(undefined, name); + } + }, [onChange, name]); + + const endDate = value?.endDate; + const endDateDate = endDate + ? dateStringToDate(endDate) + : new Date(); + + const startDate = value?.startDate; + let startDateDate = startDate + ? dateStringToDate(startDate) + : new Date(); + + if (sameMonth(endDateDate, startDateDate)) { + startDateDate = prevMonth(startDateDate); + } + + const firstInitialDate = ymdToDateString( + startDateDate.getFullYear(), + startDateDate.getMonth(), + 1, + ); + const secondInitialDate = ymdToDateString( + endDateDate.getFullYear(), + endDateDate.getMonth(), + 1, + ); + + const dateInputLabel = useMemo( + () => { + if ( + isNotDefined(tempDate.startDate) + && isNotDefined(value?.startDate) + && isNotDefined(value?.endDate) + ) { + return undefined; + } + + const startDateString = tempDate.startDate ?? value?.startDate; + const start = isDefined(startDateString) + ? new Date(startDateString).toLocaleDateString() + : '--'; + const endDateString = value?.endDate; + const end = isDefined(endDateString) + ? new Date(endDateString).toLocaleDateString() + : '--'; + + return [ + start, + end, + ].join(' to '); + }, + [value, tempDate], + ); + + return ( + <> + + { actions } + {!readOnly && ( + <> + {value && ( + + )} + + + )} + + )} + actionsContainerClassName={actionsContainerClassName} + className={className} + disabled={disabled} + error={error} + errorContainerClassName={errorContainerClassName} + hint={hint} + hintContainerClassName={hintContainerClassName} + icons={icons} + iconsContainerClassName={iconsContainerClassName} + inputSectionClassName={inputSectionClassName} + inputContainerClassName={styles.inputContainer} + label={label} + labelContainerClassName={labelContainerClassName} + readOnly={readOnly} + input={( + + )} + /> + {!readOnly && showCalendar && ( + +
+ {predefinedDateRangeOptions.map((opt) => ( + + {opt.label} + + ))} +
+ + + + + )} + + ); +} + +export default DateRangeInput; diff --git a/manager-dashboard/app/components/DateRangeInput/predefinedDateRange.ts b/manager-dashboard/app/components/DateRangeInput/predefinedDateRange.ts new file mode 100644 index 000000000..4764069c6 --- /dev/null +++ b/manager-dashboard/app/components/DateRangeInput/predefinedDateRange.ts @@ -0,0 +1,181 @@ +export type PredefinedDateRangeKey = 'today' + | 'yesterday' + | 'thisWeek' + | 'lastSevenDays' + | 'thisMonth' + | 'lastThirtyDays' + | 'lastThreeMonths' + | 'lastSixMonths' + | 'thisYear' + | 'lastYear'; + +export interface PredefinedDateRangeOption { + key: PredefinedDateRangeKey; + label: string; + getValue: () => ({ startDate: Date, endDate: Date }); +} + +export const predefinedDateRangeOptions: PredefinedDateRangeOption[] = [ + { + key: 'today', + label: 'Today', + getValue: () => ({ + startDate: new Date(), + endDate: new Date(), + }), + }, + { + key: 'yesterday', + label: 'Yesterday', + getValue: () => { + const startDate = new Date(); + startDate.setDate(startDate.getDate() - 1); + + const endDate = new Date(); + endDate.setDate(endDate.getDate() - 1); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'thisWeek', + label: 'This week', + getValue: () => { + const startDate = new Date(); + startDate.setDate(startDate.getDate() - startDate.getDay()); + + const endDate = new Date(); + // NOTE: this will give us sunday + endDate.setDate(startDate.getDate() + 6); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'lastSevenDays', + label: 'Last 7 days', + getValue: () => { + const endDate = new Date(); + + const startDate = new Date(); + startDate.setDate(endDate.getDate() - 7); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'thisMonth', + label: 'This month', + getValue: () => { + const startDate = new Date(); + startDate.setDate(1); + + const endDate = new Date(); + endDate.setMonth(endDate.getMonth() + 1); + endDate.setDate(0); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'lastThirtyDays', + label: 'Last 30 days', + getValue: () => { + const endDate = new Date(); + + const startDate = new Date(); + startDate.setDate(endDate.getDate() - 30); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'lastThreeMonths', + label: 'Last 3 months', + getValue: () => { + const startDate = new Date(); + startDate.setMonth(startDate.getMonth() - 2); + startDate.setDate(1); + + const endDate = new Date(); + endDate.setMonth(endDate.getMonth() + 1); + endDate.setDate(0); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'lastSixMonths', + label: 'Last 6 months', + getValue: () => { + const startDate = new Date(); + startDate.setMonth(startDate.getMonth() - 5); + startDate.setDate(1); + + const endDate = new Date(); + endDate.setMonth(endDate.getMonth() + 1); + endDate.setDate(0); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'thisYear', + label: 'This year', + getValue: () => { + const startDate = new Date(); + startDate.setMonth(0); + startDate.setDate(1); + + const endDate = new Date(); + endDate.setFullYear(startDate.getFullYear() + 1); + endDate.setMonth(0); + endDate.setDate(0); + + return { + startDate, + endDate, + }; + }, + }, + { + key: 'lastYear', + label: 'Last year', + getValue: () => { + const startDate = new Date(); + startDate.setFullYear(startDate.getFullYear() - 1); + startDate.setMonth(0); + startDate.setDate(1); + + const endDate = new Date(); + endDate.setMonth(0); + endDate.setDate(0); + + return { + startDate, + endDate, + }; + }, + }, +]; diff --git a/manager-dashboard/app/components/DateRangeInput/styles.css b/manager-dashboard/app/components/DateRangeInput/styles.css new file mode 100644 index 000000000..4d4388b75 --- /dev/null +++ b/manager-dashboard/app/components/DateRangeInput/styles.css @@ -0,0 +1,85 @@ +.input-container { + display: flex; + flex-direction: row; + + .input { + --color: var(--color-text); + flex-grow: 1; + min-width: unset; + color: var(--color); + + &.empty { + --color: var(--color-input-placeholder); /* TODO */ + } + + &.errored { + --color: var(--color-danger); + } + } +} + +.calendar-popup { + height: 25rem; + + .popup-content { + display: flex; + max-width: unset!important; + max-height: unset!important; + + .calendar { + --padding: var(--spacing-medium); + width: calc(var(--width-calendar-date) * 7 + 2 * var(--padding)); + height: 100%; + } + + .predefined-options { + display: flex; + flex-direction: column; + justify-content: center; + padding: calc(var(--spacing-medium) - var(--spacing-small)); + + .option { + padding: var(--spacing-small); + width: 100%; + text-align: right; + + &:hover { + background-color: var(--color-background-hover-light); + } + } + } + } +} + +.calendar-date { + &.start-date { + border-top-right-radius: 0; + border-bottom-right-radius: 0; + + &:not(.ghost) { + background-color: var(--color-accent); + color: var(--color-text-on-dark); + } + &.ghost { + background-color: var(--color-background-hover-light); + } + } + + &.end-date { + border-top-left-radius: 0; + border-bottom-left-radius: 0; + + &:not(.ghost) { + background-color: var(--color-accent); + color: var(--color-text-ligth); + } + &.ghost { + background-color: var(--color-background-hover-light); + } + } + + &.in-between { + border-radius: 0; + background-color: var(--color-background-hover-light); + } +} diff --git a/manager-dashboard/app/utils/common.tsx b/manager-dashboard/app/utils/common.tsx index 571be709c..53338d34f 100644 --- a/manager-dashboard/app/utils/common.tsx +++ b/manager-dashboard/app/utils/common.tsx @@ -65,8 +65,9 @@ export const PROJECT_TYPE_BUILD_AREA = 1; export const PROJECT_TYPE_FOOTPRINT = 2; export const PROJECT_TYPE_CHANGE_DETECTION = 3; export const PROJECT_TYPE_COMPLETENESS = 4; +export const PROJECT_TYPE_STREET = 7; -export type ProjectType = 1 | 2 | 3 | 4; +export type ProjectType = 1 | 2 | 3 | 4 | 7; export const projectTypeLabelMap: { [key in ProjectType]: string @@ -75,6 +76,7 @@ export const projectTypeLabelMap: { [PROJECT_TYPE_FOOTPRINT]: 'Validate', [PROJECT_TYPE_CHANGE_DETECTION]: 'Compare', [PROJECT_TYPE_COMPLETENESS]: 'Completeness', + [PROJECT_TYPE_STREET]: 'Street', }; export type IconKey = 'add-outline' @@ -321,3 +323,17 @@ export const formatProjectTopic = (projectTopic: string) => { return newProjectTopic; }; + +export function ymdToDateString(year: number, month: number, day: number) { + const ys = String(year).padStart(4, '0'); + const ms = String(month + 1).padStart(2, '0'); + const ds = String(day).padStart(2, '0'); + + return `${ys}-${ms}-${ds}`; +} + +export function dateStringToDate(value: string) { + return new Date(`${value}T00:00`); +} + +export const typedMemo: ((c: T) => T) = React.memo; diff --git a/manager-dashboard/app/views/NewProject/index.tsx b/manager-dashboard/app/views/NewProject/index.tsx index f63bbc065..811279723 100644 --- a/manager-dashboard/app/views/NewProject/index.tsx +++ b/manager-dashboard/app/views/NewProject/index.tsx @@ -50,6 +50,9 @@ import Button from '#components/Button'; import NonFieldError from '#components/NonFieldError'; import AnimatedSwipeIcon from '#components/AnimatedSwipeIcon'; import ExpandableContainer from '#components/ExpandableContainer'; +import AlertBanner from '#components/AlertBanner'; +import Checkbox from '#components/Checkbox'; +import DateRangeInput from '#components/DateRangeInput'; import { valueSelector, labelSelector, @@ -59,6 +62,7 @@ import { PROJECT_TYPE_FOOTPRINT, PROJECT_TYPE_COMPLETENESS, PROJECT_TYPE_CHANGE_DETECTION, + PROJECT_TYPE_STREET, formatProjectTopic, } from '#utils/common'; import { getValueFromFirebase } from '#utils/firebase'; @@ -104,6 +108,8 @@ const defaultProjectFormValue: PartialProjectFormType = { // maxTasksPerUser: -1, inputType: PROJECT_INPUT_TYPE_UPLOAD, filter: FILTER_BUILDINGS, + randomizeOrder: false, + panoOnly: false, }; interface Props { @@ -313,6 +319,10 @@ function NewProject(props: Props) { valuesToCopy.geometry = res.geometry; } + valuesToCopy.startTimestamp = valuesToCopy.dateRange?.startDate ?? null; + valuesToCopy.endTimestamp = valuesToCopy.dateRange?.endDate ?? null; + valuesToCopy.isPano = valuesToCopy.panoOnly ? true : null; + const storage = getStorage(); const timestamp = (new Date()).getTime(); const uploadedImageRef = storageRef(storage, `projectImages/${timestamp}-project-image-${projectImage.name}`); @@ -417,6 +427,11 @@ function NewProject(props: Props) { || projectSubmissionStatus === 'projectSubmit' ); + const tileServerVisible = value.projectType === PROJECT_TYPE_BUILD_AREA + || value.projectType === PROJECT_TYPE_FOOTPRINT + || value.projectType === PROJECT_TYPE_COMPLETENESS + || value.projectType === PROJECT_TYPE_CHANGE_DETECTION; + const tileServerBVisible = value.projectType === PROJECT_TYPE_CHANGE_DETECTION || value.projectType === PROJECT_TYPE_COMPLETENESS; @@ -459,6 +474,16 @@ function NewProject(props: Props) { error={error?.projectType} disabled={submissionPending || testPending} /> + {value.projectType === PROJECT_TYPE_STREET && ( + +
+
+ Projects of this type are currently + only visible in the web app. +
+
+ + )} {( - value.projectType === PROJECT_TYPE_FOOTPRINT + (value.projectType === PROJECT_TYPE_FOOTPRINT + || value.projectType === PROJECT_TYPE_STREET) && customOptions && customOptions.length > 0 ) && ( @@ -526,7 +552,8 @@ function NewProject(props: Props) { )} {(value.projectType === PROJECT_TYPE_BUILD_AREA || value.projectType === PROJECT_TYPE_CHANGE_DETECTION - || value.projectType === PROJECT_TYPE_COMPLETENESS) && ( + || value.projectType === PROJECT_TYPE_COMPLETENESS + || value.projectType === PROJECT_TYPE_STREET) && ( @@ -661,17 +688,19 @@ function NewProject(props: Props) { /> - - - + {tileServerVisible && ( + + + + )} {tileServerBVisible && ( )} + + {value.projectType === PROJECT_TYPE_STREET && ( + + + + +
+ + + +
+ + )} + {error?.[nonFieldError] && (
{error?.[nonFieldError]} diff --git a/manager-dashboard/app/views/NewProject/styles.css b/manager-dashboard/app/views/NewProject/styles.css index bcbad1ed6..cbfa76230 100644 --- a/manager-dashboard/app/views/NewProject/styles.css +++ b/manager-dashboard/app/views/NewProject/styles.css @@ -75,6 +75,18 @@ flex-shrink: 0; justify-content: center; } + + .warning-container { + display: flex; + flex-direction: column; + gap: var(--spacing-extra-small); + + .warning-item { + display: flex; + gap: var(--spacing-small); + align-items: flex-start; + } + } } .submission-status-modal { diff --git a/manager-dashboard/app/views/NewProject/utils.ts b/manager-dashboard/app/views/NewProject/utils.ts index e607d3a15..685ffd39d 100644 --- a/manager-dashboard/app/views/NewProject/utils.ts +++ b/manager-dashboard/app/views/NewProject/utils.ts @@ -26,6 +26,8 @@ import { tileServerFieldsSchema, } from '#components/TileServerInput'; +import { Value as DateRange } from '#components/DateRangeInput'; + import { getNoMoreThanNCharacterCondition, ProjectType, @@ -34,6 +36,7 @@ import { PROJECT_TYPE_FOOTPRINT, PROJECT_TYPE_CHANGE_DETECTION, PROJECT_TYPE_COMPLETENESS, + PROJECT_TYPE_STREET, IconKey, } from '#utils/common'; @@ -75,6 +78,15 @@ export interface ProjectFormType { tileServer: TileServer; tileServerB?: TileServer; customOptions?: CustomOptionsForProject; + dateRange?: DateRange | null; + startTimestamp?: string | null; + endTimestamp?: string | null; + organizationId?: number; + creatorId?: number; + randomizeOrder?: boolean; + panoOnly?: boolean; + isPano?: boolean | null; + samplingThreshold?: number; } export const PROJECT_INPUT_TYPE_UPLOAD = 'aoi_file'; @@ -272,6 +284,38 @@ export const projectFormSchema: ProjectFormSchema = { greaterThanCondition(0), ], }, + dateRange: { + required: false, + }, + creatorId: { + required: false, + validations: [ + integerCondition, + greaterThanCondition(0), + ], + }, + organizationId: { + required: false, + validations: [ + integerCondition, + greaterThanCondition(0), + ], + }, + samplingThreshold: { + required: false, + validation: [ + greaterThanCondition(0), + ], + }, + panoOnly: { + required: false, + }, + isPano: { + required: false, + }, + randomizeOrder: { + required: false, + }, }; baseSchema = addCondition( @@ -280,7 +324,8 @@ export const projectFormSchema: ProjectFormSchema = { ['projectType'], ['customOptions'], (formValues) => { - if (formValues?.projectType === PROJECT_TYPE_FOOTPRINT) { + if (formValues?.projectType === PROJECT_TYPE_FOOTPRINT + || formValues?.projectType === PROJECT_TYPE_STREET) { return { customOptions: { keySelector: (key) => key.value, @@ -394,6 +439,7 @@ export const projectFormSchema: ProjectFormSchema = { projectType === PROJECT_TYPE_BUILD_AREA || projectType === PROJECT_TYPE_COMPLETENESS || projectType === PROJECT_TYPE_CHANGE_DETECTION + || projectType === PROJECT_TYPE_STREET || (projectType === PROJECT_TYPE_FOOTPRINT && ( inputType === PROJECT_INPUT_TYPE_UPLOAD )) @@ -541,7 +587,9 @@ export function getGroupSize(projectType: ProjectType | undefined) { return 120; } - if (projectType === PROJECT_TYPE_FOOTPRINT || projectType === PROJECT_TYPE_CHANGE_DETECTION) { + if (projectType === PROJECT_TYPE_FOOTPRINT + || projectType === PROJECT_TYPE_CHANGE_DETECTION + || projectType === PROJECT_TYPE_STREET) { return 25; } diff --git a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx index 607434590..b309be7ff 100644 --- a/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx +++ b/manager-dashboard/app/views/NewTutorial/ScenarioPageInput/index.tsx @@ -17,6 +17,7 @@ import { PROJECT_TYPE_FOOTPRINT, PROJECT_TYPE_CHANGE_DETECTION, PROJECT_TYPE_COMPLETENESS, + PROJECT_TYPE_STREET, } from '#utils/common'; import TextInput from '#components/TextInput'; import Heading from '#components/Heading'; @@ -318,7 +319,14 @@ export default function ScenarioPageInput(props: Props) { lookFor={lookFor} /> )} - {(projectType && projectType !== PROJECT_TYPE_FOOTPRINT) && ( + {projectType === PROJECT_TYPE_STREET && ( +
+ Preview not available. +
+ )} + {(projectType + && projectType !== PROJECT_TYPE_FOOTPRINT + && projectType !== PROJECT_TYPE_STREET) && ( checkSchema( @@ -341,7 +347,6 @@ const defaultTutorialFormValue: PartialTutorialFormType = { name: TILE_SERVER_ESRI, credits: tileServerDefaultCredits[TILE_SERVER_ESRI], }, - customOptions: defaultFootprintCustomOptions, }; type SubmissionStatus = 'started' | 'imageUpload' | 'tutorialSubmit' | 'success' | 'failed'; @@ -646,6 +651,11 @@ function NewTutorial(props: Props) { || tutorialSubmissionStatus === 'tutorialSubmit' ); + const tileServerVisible = value.projectType === PROJECT_TYPE_BUILD_AREA + || value.projectType === PROJECT_TYPE_FOOTPRINT + || value.projectType === PROJECT_TYPE_COMPLETENESS + || value.projectType === PROJECT_TYPE_CHANGE_DETECTION; + const tileServerBVisible = value.projectType === PROJECT_TYPE_CHANGE_DETECTION || value.projectType === PROJECT_TYPE_COMPLETENESS; @@ -716,6 +726,7 @@ function NewTutorial(props: Props) { setFieldValue(undefined, 'tutorialTasks'); setFieldValue(undefined, 'scenarioPages'); setFieldValue(newValue, 'projectType'); + setFieldValue(getDefaultOptions(newValue), 'customOptions'); }, [setFieldValue], ); @@ -761,7 +772,10 @@ function NewTutorial(props: Props) { autoFocus /> - {value.projectType === PROJECT_TYPE_FOOTPRINT && ( + {( + value.projectType === PROJECT_TYPE_FOOTPRINT + || value.projectType === PROJECT_TYPE_STREET + ) && ( - - - + {tileServerVisible && ( + + + + )} + {tileServerBVisible && ( ( value: T, key: K, @@ -268,6 +296,18 @@ export function deleteKey( return copy; } +export function getDefaultOptions(projectType: ProjectType | undefined) { + if (projectType === PROJECT_TYPE_FOOTPRINT) { + return defaultFootprintCustomOptions; + } + + if (projectType === PROJECT_TYPE_STREET) { + return defaultStreetCustomOptions; + } + + return undefined; +} + export interface BuildAreaProperties { reference: number; screen: number; @@ -308,6 +348,12 @@ export interface ChangeDetectionProperties { // taskId: string; } +export interface StreetProperties { + id: string; + reference: number; + screen: number; +} + export type BuildAreaGeoJSON = GeoJSON.FeatureCollection< GeoJSON.Geometry, BuildAreaProperties @@ -323,9 +369,14 @@ export type ChangeDetectionGeoJSON = GeoJSON.FeatureCollection< ChangeDetectionProperties >; +export type StreetGeoJSON = GeoJSON.FeatureCollection< + GeoJSON.Geometry, + StreetProperties +>; + export type TutorialTasksGeoJSON = GeoJSON.FeatureCollection< GeoJSON.Geometry, - BuildAreaProperties | FootprintProperties | ChangeDetectionProperties + BuildAreaProperties | FootprintProperties | ChangeDetectionProperties | StreetProperties >; export type CustomOptions = { @@ -724,7 +775,8 @@ export const tutorialFormSchema: TutorialFormSchema = { }), }; - if (formValues?.projectType === PROJECT_TYPE_FOOTPRINT) { + if (formValues?.projectType === PROJECT_TYPE_FOOTPRINT + || formValues?.projectType === PROJECT_TYPE_STREET) { return { customOptions: customOptionField, }; diff --git a/mapswipe_workers/mapswipe_workers/definitions.py b/mapswipe_workers/mapswipe_workers/definitions.py index bb8c7296f..aa32d3aac 100644 --- a/mapswipe_workers/mapswipe_workers/definitions.py +++ b/mapswipe_workers/mapswipe_workers/definitions.py @@ -16,6 +16,8 @@ OSM_API_LINK = "https://www.openstreetmap.org/api/0.6/" OSMCHA_API_LINK = "https://osmcha.org/api/v1/" OSMCHA_API_KEY = os.environ["OSMCHA_API_KEY"] +MAPILLARY_API_LINK = "https://tiles.mapillary.com/maps/vtp/mly1_computed_public/2/" +MAPILLARY_API_KEY = os.environ["MAPILLARY_API_KEY"] # number of geometries for project geometries MAX_INPUT_GEOMETRIES = 10 @@ -134,6 +136,7 @@ class ProjectType(Enum): COMPLETENESS = 4 MEDIA_CLASSIFICATION = 5 DIGITIZATION = 6 + STREET = 7 @property def constructor(self): @@ -145,6 +148,7 @@ def constructor(self): DigitizationProject, FootprintProject, MediaClassificationProject, + StreetProject, ) project_type_classes = { @@ -154,6 +158,7 @@ def constructor(self): 4: CompletenessProject, 5: MediaClassificationProject, 6: DigitizationProject, + 7: StreetProject, } return project_type_classes[self.value] @@ -165,6 +170,7 @@ def tutorial(self): ClassificationTutorial, CompletenessTutorial, FootprintTutorial, + StreetTutorial, ) project_type_classes = { @@ -172,5 +178,6 @@ def tutorial(self): 2: FootprintTutorial, 3: ChangeDetectionTutorial, 4: CompletenessTutorial, + 7: StreetTutorial, } return project_type_classes[self.value] diff --git a/mapswipe_workers/mapswipe_workers/project_types/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/__init__.py index a07ff38be..9560c76ef 100644 --- a/mapswipe_workers/mapswipe_workers/project_types/__init__.py +++ b/mapswipe_workers/mapswipe_workers/project_types/__init__.py @@ -2,6 +2,8 @@ from .arbitrary_geometry.footprint.project import FootprintProject from .arbitrary_geometry.footprint.tutorial import FootprintTutorial from .media_classification.project import MediaClassificationProject +from .street.project import StreetProject +from .street.tutorial import StreetTutorial from .tile_map_service.change_detection.project import ChangeDetectionProject from .tile_map_service.change_detection.tutorial import ChangeDetectionTutorial from .tile_map_service.classification.project import ClassificationProject @@ -20,4 +22,6 @@ "FootprintProject", "FootprintTutorial", "DigitizationProject", + "StreetProject", + "StreetTutorial", ] diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/__init__.py b/mapswipe_workers/mapswipe_workers/project_types/street/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/project.py b/mapswipe_workers/mapswipe_workers/project_types/street/project.py new file mode 100644 index 000000000..4d0fe8c5b --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/street/project.py @@ -0,0 +1,136 @@ +import math +from dataclasses import dataclass +from typing import Dict, List + +from mapswipe_workers.definitions import logger +from mapswipe_workers.firebase.firebase import Firebase +from mapswipe_workers.firebase_to_postgres.transfer_results import ( + results_to_file, + save_results_to_postgres, + truncate_temp_results, +) +from mapswipe_workers.generate_stats.project_stats import ( + get_statistics_for_integer_result_project, +) +from mapswipe_workers.project_types.project import BaseGroup, BaseProject, BaseTask +from mapswipe_workers.utils.process_mapillary import get_image_metadata +from mapswipe_workers.utils.validate_input import ( + build_multipolygon_from_layer_geometries, + check_if_layer_has_too_many_geometries, + check_if_layer_is_empty, + load_geojson_to_ogr, + multipolygon_to_wkt, + save_geojson_to_file, +) + + +@dataclass +class StreetGroup(BaseGroup): + # todo: does client use this, or only for the implementation of project creation? + pass + + +@dataclass +class StreetTask(BaseTask): + geometry: str + + +class StreetProject(BaseProject): + def __init__(self, project_draft): + super().__init__(project_draft) + self.groups: Dict[str, StreetGroup] = {} + self.tasks: Dict[str, List[StreetTask]] = {} + + self.geometry = project_draft["geometry"] + + # TODO: validate inputs + ImageMetadata = get_image_metadata( + self.geometry, + creator_id=project_draft.get("creatorId", None), + is_pano=project_draft.get("isPano", None), + start_time=project_draft.get("startTimestamp", None), + end_time=project_draft.get("endTimestamp", None), + organization_id=project_draft.get("organizationId", None), + randomize_order=project_draft.get("randomizeOrder", None), + sampling_threshold=project_draft.get("samplingThreshold", None), + ) + + self.imageIds = ImageMetadata["ids"] + self.imageGeometries = ImageMetadata["geometries"] + + def save_tasks_to_firebase(self, projectId: str, tasks: dict): + firebase = Firebase() + firebase.save_tasks_to_firebase(projectId, tasks, useCompression=False) + + @staticmethod + def results_to_postgres(results: dict, project_id: str, filter_mode: bool): + """How to move the result data from firebase to postgres.""" + results_file, user_group_results_file = results_to_file(results, project_id) + truncate_temp_results() + save_results_to_postgres(results_file, project_id, filter_mode) + return user_group_results_file + + @staticmethod + def get_per_project_statistics(project_id, project_info): + """How to aggregate the project results.""" + return get_statistics_for_integer_result_project( + project_id, project_info, generate_hot_tm_geometries=False + ) + + def validate_geometries(self): + self.inputGeometriesFileName = save_geojson_to_file( + self.projectId, self.geometry + ) + layer, datasource = load_geojson_to_ogr( + self.projectId, self.inputGeometriesFileName + ) + + # check if inputs fit constraints + check_if_layer_is_empty(self.projectId, layer) + + multi_polygon, project_area = build_multipolygon_from_layer_geometries( + self.projectId, layer + ) + + check_if_layer_has_too_many_geometries(self.projectId, multi_polygon) + + del datasource + del layer + + logger.info( + f"{self.projectId}" f" - validate geometry - " f"input geometry is correct." + ) + wkt_geometry = multipolygon_to_wkt(multi_polygon) + return wkt_geometry + + def create_groups(self): + self.numberOfGroups = math.ceil(len(self.imageIds) / self.groupSize) + for group_id in range(self.numberOfGroups): + self.groups[f"g{group_id}"] = StreetGroup( + projectId=self.projectId, + groupId=f"g{group_id}", + progress=0, + finishedCount=0, + requiredCount=0, + numberOfTasks=self.groupSize, + ) + + def create_tasks(self): + if len(self.groups) == 0: + raise ValueError("Groups needs to be created before tasks can be created.") + for group_id, group in self.groups.items(): + self.tasks[group_id] = [] + for i in range(self.groupSize): + task = StreetTask( + projectId=self.projectId, + groupId=group_id, + geometry=self.imageGeometries.pop(), + taskId=self.imageIds.pop(), + ) + self.tasks[group_id].append(task) + + # list now empty? if usual group size is not reached + # the actual number of tasks for the group is updated + if not self.imageIds: + group.numberOfTasks = i + 1 + break diff --git a/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py new file mode 100644 index 000000000..e59a97f09 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/project_types/street/tutorial.py @@ -0,0 +1,84 @@ +from dataclasses import asdict, dataclass + +from mapswipe_workers.definitions import logger +from mapswipe_workers.firebase.firebase import Firebase +from mapswipe_workers.project_types.street.project import StreetGroup, StreetTask +from mapswipe_workers.project_types.tutorial import BaseTutorial + + +@dataclass +class StreetTutorialTask(StreetTask): + projectId: int + taskId: str + groupId: int + referenceAnswer: int + screen: int + + +class StreetTutorial(BaseTutorial): + """The subclass for an arbitrary geometry based Tutorial.""" + + def __init__(self, tutorial_draft): + # this will create the basis attributes + super().__init__(tutorial_draft) + + # self.projectId = tutorial_draft["projectId"] + self.projectType = tutorial_draft["projectType"] + self.tutorial_tasks = tutorial_draft["tutorialTasks"] + self.groups = dict() + self.tasks = dict() + + def create_tutorial_groups(self): + """Create group for the tutorial based on provided examples in geojson file.""" + # load examples/tasks from file + + group = StreetGroup( + groupId=101, + projectId=self.projectId, + numberOfTasks=len(self.tutorial_tasks), + progress=0, + finishedCount=0, + requiredCount=0, + ) + self.groups[101] = group + + # Add number of tasks for the group here. This needs to be set according to + # the number of features/examples in the geojson file + + logger.info( + f"{self.projectId}" + f" - create_tutorial_groups - " + f"created groups dictionary" + ) + + def create_tutorial_tasks(self): + """Create the tasks dict based on provided examples in geojson file.""" + task_list = [] + for i, task in enumerate(self.tutorial_tasks["features"]): + task = StreetTutorialTask( + projectId=self.projectId, + groupId=101, + taskId=f"{task['properties']['id']}", + geometry="", + referenceAnswer=task["properties"]["reference"], + screen=task["properties"]["screen"], + ) + task_list.append(asdict(task)) + if task_list: + self.tasks[101] = task_list + else: + logger.info(f"group in project {self.projectId} is not valid.") + + logger.info( + f"{self.projectId}" + f" - create_tutorial_tasks - " + f"created tasks dictionary" + ) + + def save_tutorial(self): + firebase = Firebase() + firebase.save_tutorial_to_firebase( + self, self.groups, self.tasks, useCompression=True + ) + logger.info(self.tutorialDraftId) + firebase.drop_tutorial_draft(self.tutorialDraftId) diff --git a/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py new file mode 100644 index 000000000..54afc2be6 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/utils/process_mapillary.py @@ -0,0 +1,249 @@ +import os +from concurrent.futures import ProcessPoolExecutor +from functools import partial + +import mercantile +import pandas as pd +import requests +from shapely import MultiPolygon, Point, Polygon, box, unary_union +from shapely.geometry import shape +from vt2geojson import tools as vt2geojson_tools + +from mapswipe_workers.definitions import ( + MAPILLARY_API_KEY, + MAPILLARY_API_LINK, + CustomError, + logger, +) +from mapswipe_workers.utils.spatial_sampling import spatial_sampling + + +def create_tiles(polygon, level): + if not isinstance(polygon, (Polygon, MultiPolygon)): + return pd.DataFrame(columns=["x", "y", "z", "geometry"]) + if isinstance(polygon, Polygon): + polygon = MultiPolygon([polygon]) + + tiles = set() + for i, poly in enumerate(polygon.geoms): + tiles.update(list(mercantile.tiles(*poly.bounds, level))) + + bbox_list = [mercantile.bounds(tile.x, tile.y, tile.z) for tile in tiles] + bbox_polygons = [box(*bbox) for bbox in bbox_list] + tiles = pd.DataFrame( + { + "x": [tile.x for tile in tiles], + "y": [tile.y for tile in tiles], + "z": [tile.z for tile in tiles], + "geometry": bbox_polygons, + } + ) + + return tiles + + +def download_and_process_tile(row, polygon, kwargs, attempt_limit=3): + z = row["z"] + x = row["x"] + y = row["y"] + url = f"{MAPILLARY_API_LINK}{z}/{x}/{y}?access_token={MAPILLARY_API_KEY}" + + attempt = 0 + while attempt < attempt_limit: + try: + data = get_mapillary_data(url, x, y, z) + if data.isna().all().all() is False or data.empty is False: + data = data[data["geometry"].apply(lambda point: point.within(polygon))] + target_columns = [ + "id", + "geometry", + "captured_at", + "is_pano", + "compass_angle", + "sequence", + "organization_id", + ] + for col in target_columns: + if col not in data.columns: + data[col] = None + if data.isna().all().all() is False or data.empty is False: + data = filter_results(data, **kwargs) + + return data + except Exception as e: + print(f"An exception occurred while requesting a tile: {e}") + attempt += 1 + + print(f"A tile could not be downloaded: {row}") + return None + + +def get_mapillary_data(url, x, y, z): + r = requests.get(url) + assert r.status_code == 200, r.content + features = vt2geojson_tools.vt_bytes_to_geojson(r.content, x, y, z).get( + "features", [] + ) + data = [] + data.extend( + [ + { + "geometry": Point(feature["geometry"]["coordinates"]), + **feature.get("properties", {}), + } + for feature in features + if feature.get("geometry", {}).get("type") == "Point" + ] + ) + return pd.DataFrame(data) + + +def coordinate_download( + polygon, level, kwargs: dict, use_concurrency=True, workers=os.cpu_count() * 4 +): + tiles = create_tiles(polygon, level) + + downloaded_metadata = [] + + if not tiles.empty: + if not use_concurrency: + workers = 1 + + downloaded_metadata = parallelized_processing( + downloaded_metadata, kwargs, polygon, tiles, workers + ) + if len(downloaded_metadata): + downloaded_metadata = pd.concat(downloaded_metadata, ignore_index=True) + else: + return pd.DataFrame(downloaded_metadata) + + return downloaded_metadata + + +def parallelized_processing(data, kwargs, polygon, tiles, workers): + process_tile_with_args = partial( + download_and_process_tile, polygon=polygon, kwargs=kwargs + ) + with ProcessPoolExecutor(max_workers=workers) as executor: + futures = list( + executor.map(process_tile_with_args, tiles.to_dict(orient="records")) + ) + + for df in futures: + if df is not None and not df.empty: + data.append(df) + return data + + +def geojson_to_polygon(geojson_data): + if geojson_data["type"] == "FeatureCollection": + features = geojson_data["features"] + elif geojson_data["type"] == "Feature": + features = [geojson_data] + else: + raise ValueError("Unsupported GeoJSON type.") + + polygons = [] + for feature in features: + geometry = shape(feature["geometry"]) + if isinstance(geometry, (Polygon, MultiPolygon)): + polygons.append(geometry) + else: + raise ValueError( + "Non-polygon geometries cannot be combined into a MultiPolygon." + ) + + combined_multipolygon = unary_union(polygons) + + return combined_multipolygon + + +def filter_by_timerange(df: pd.DataFrame, start_time: str, end_time: str = None): + df["captured_at"] = pd.to_datetime(df["captured_at"], unit="ms") + start_time = pd.to_datetime(start_time).tz_localize(None) + if end_time is None: + end_time = pd.Timestamp.now().tz_localize(None) + else: + end_time = pd.to_datetime(end_time).tz_localize(None) + filtered_df = df[ + (df["captured_at"] >= start_time) & (df["captured_at"] <= end_time) + ] + return filtered_df + + +def filter_results( + results_df: pd.DataFrame, + creator_id: int = None, + is_pano: bool = None, + organization_id: str = None, + start_time: str = None, + end_time: str = None, +): + df = results_df.copy() + if creator_id is not None: + if df["creator_id"].isna().all(): + logger.info("No Mapillary Feature in the AoI has a 'creator_id' value.") + return None + df = df[df["creator_id"] == creator_id] + if is_pano is not None: + if df["is_pano"].isna().all(): + logger.info("No Mapillary Feature in the AoI has a 'is_pano' value.") + return None + df = df[df["is_pano"] == is_pano] + if organization_id is not None: + if df["organization_id"].isna().all(): + logger.info( + "No Mapillary Feature in the AoI has an 'organization_id' value." + ) + return None + df = df[df["organization_id"] == organization_id] + if start_time is not None: + if df["captured_at"].isna().all(): + logger.info("No Mapillary Feature in the AoI has a 'captured_at' value.") + return None + df = filter_by_timerange(df, start_time, end_time) + return df + + +def get_image_metadata( + aoi_geojson, + level=14, + is_pano: bool = None, + creator_id: int = None, + organization_id: str = None, + start_time: str = None, + end_time: str = None, + randomize_order=False, + sampling_threshold=None, +): + kwargs = { + "is_pano": is_pano, + "creator_id": creator_id, + "organization_id": organization_id, + "start_time": start_time, + "end_time": end_time, + } + aoi_polygon = geojson_to_polygon(aoi_geojson) + downloaded_metadata = coordinate_download(aoi_polygon, level, kwargs) + if downloaded_metadata.empty or downloaded_metadata.isna().all().all(): + raise CustomError( + "No Mapillary Features in the AoI or no Features match the filter criteria." + ) + if sampling_threshold is not None: + downloaded_metadata = spatial_sampling(downloaded_metadata, sampling_threshold) + + if randomize_order is True: + downloaded_metadata = downloaded_metadata.sample(frac=1).reset_index(drop=True) + + downloaded_metadata = downloaded_metadata.drop_duplicates(subset=["geometry"]) + + total_images = len(downloaded_metadata) + if total_images > 100000: + raise CustomError( + f"Too many Images with selected filter options for the AoI: {total_images}" + ) + + return { + "ids": downloaded_metadata["id"].tolist(), + "geometries": downloaded_metadata["geometry"].tolist(), + } diff --git a/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py new file mode 100644 index 000000000..67f35c7e9 --- /dev/null +++ b/mapswipe_workers/mapswipe_workers/utils/spatial_sampling.py @@ -0,0 +1,155 @@ +import numpy as np +import pandas as pd + + +def distance_on_sphere(p1, p2): + """ + p1 and p2 are two lists that have two elements. They are numpy arrays of the long + and lat coordinates of the points in set1 and set2 + + Calculate the distance between two points on the Earth's surface using the + haversine formula. + + Args: + p1 (list): Array containing the longitude and latitude coordinates of points + FROM which the distance to be calculated in degree + p2 (list): Array containing the longitude and latitude coordinates of points + TO which the distance to be calculated in degree + + Returns: + numpy.ndarray: Array containing the distances between the two points on the + sphere in kilometers. + + This function computes the distance between two points on the Earth's surface + using the haversine formula, which takes into account the spherical shape of the + Earth. The input arrays `p1` and `p2` should contain longitude and latitude + coordinates in degrees. The function returns an array containing the distances + between corresponding pairs of points. + """ + earth_radius = 6371 # km + + p1 = np.radians(np.array(p1)) + p2 = np.radians(np.array(p2)) + + delta_lat = p2[1] - p1[1] + delta_long = p2[0] - p1[0] + + a = ( + np.sin(delta_lat / 2) ** 2 + + np.cos(p1[1]) * np.cos(p2[1]) * np.sin(delta_long / 2) ** 2 + ) + c = 2 * np.arcsin(np.sqrt(a)) + + distances = earth_radius * c + return distances + + +"""----------------------------Filtering Points-------------------------------""" + + +def filter_points(df, threshold_distance): + """ + Filter points from a DataFrame based on a threshold distance. + + Args: + df (pandas.DataFrame): DataFrame containing latitude and longitude columns. + threshold_distance (float): Threshold distance for filtering points in kms. + + Returns: + pandas.DataFrame: Filtered DataFrame containing selected points. + float: Total road length calculated from the selected points. + + This function filters points from a DataFrame based on the given threshold + distance. It calculates distances between consecutive points and accumulates them + until the accumulated distance surpasses the threshold distance. It then selects + those points and constructs a new DataFrame. Additionally, it manually checks the + last point to include it if it satisfies the length condition. The function + returns the filtered DataFrame along with the calculated road length. + """ + road_length = 0 + mask = np.zeros(len(df), dtype=bool) + mask[0] = True + lat = df["lat"].to_numpy() + long = df["long"].to_numpy() + + distances = distance_on_sphere([long[1:], lat[1:]], [long[:-1], lat[:-1]]) + road_length = np.sum(distances) + + # save the last point if the road segment is relavitely small (< 2*road_length) + if threshold_distance <= road_length < 2 * threshold_distance: + mask[-1] = True + + accumulated_distance = 0 + for i, distance in enumerate(distances): + accumulated_distance += distance + if accumulated_distance >= threshold_distance: + mask[i + 1] = True + accumulated_distance = 0 # Reset accumulated distance + + to_be_returned_df = df[mask] + # since the last point has to be omitted in the vectorized distance calculation, + # it is being checked manually + p2 = to_be_returned_df.iloc[0] + distance = distance_on_sphere( + [float(p2["long"]), float(p2["lat"])], [long[-1], lat[-1]] + ) + + # last point will be added if it suffices the length condition + # last point will be added in case there is only one point returned + if distance >= threshold_distance or len(to_be_returned_df) == 1: + to_be_returned_df = pd.concat( + [ + to_be_returned_df, + pd.DataFrame(df.iloc[-1], columns=to_be_returned_df.columns), + ], + axis=0, + ) + return to_be_returned_df + + +def spatial_sampling(df, interval_length): + """ + Calculate spacing between points in a GeoDataFrame. + + Args: + df (pandas.DataFrame): DataFrame containing points with timestamps. + interval_length (float): Interval length for filtering points in kms. + + Returns: + geopandas.GeoDataFrame: Filtered GeoDataFrame containing selected points. + float: Total road length calculated from the selected points. + + This function calculates the spacing between points in a GeoDataFrame by filtering + points based on the provided interval length. It first sorts the GeoDataFrame by + timestamp and then filters points using the filter_points function. The function + returns the filtered GeoDataFrame along with the total road length. + """ + if len(df) == 1: + return df + + df["long"] = df["geometry"].apply( + lambda geom: geom.x if geom.geom_type == "Point" else None + ) + df["lat"] = df["geometry"].apply( + lambda geom: geom.y if geom.geom_type == "Point" else None + ) + sorted_df = df.sort_values(by=["captured_at"]) + + sampled_sequence_df = pd.DataFrame() + + # loop through each sequence + for sequence in sorted_df["sequence_id"].unique(): + sequence_df = sorted_df[sorted_df["sequence_id"] == sequence] + + if interval_length: + sequence_df = filter_points(sequence_df, interval_length) + if "is_pano" in sequence_df.columns: + # below line prevents FutureWarning + # (https://stackoverflow.com/questions/73800841/add-series-as-a-new-row-into-dataframe-triggers-futurewarning) + sequence_df["is_pano"] = sequence_df["is_pano"].astype(bool) + sampled_sequence_df = pd.concat([sampled_sequence_df, sequence_df], axis=0) + + # reverse order such that sequence are in direction of travel + sampled_sequence_df = sampled_sequence_df.iloc[::-1] + + return sampled_sequence_df diff --git a/mapswipe_workers/requirements.txt b/mapswipe_workers/requirements.txt index 7e125fe4b..588754060 100644 --- a/mapswipe_workers/requirements.txt +++ b/mapswipe_workers/requirements.txt @@ -14,3 +14,6 @@ sentry-sdk==0.18.0 six==1.15.0 slackclient==2.9.2 xdg==4.0.1 +shapely +mercantile +vt2geojson diff --git a/mapswipe_workers/sample_data/street/README.md b/mapswipe_workers/sample_data/street/README.md new file mode 100644 index 000000000..dfa52293e --- /dev/null +++ b/mapswipe_workers/sample_data/street/README.md @@ -0,0 +1,58 @@ +# Creating a New 'Street' Tutorial +### Useful Links +- MapSwipe Development Server: [https://dev-managers.mapswipe.org] +- MapSwipe Development App Installation Guide: [https://github.com/mapswipe/mapswipe/wiki/How-to-test-the-development-version-of-MapSwipe](https://github.com/mapswipe/mapswipe/wiki/How-to-test-the-development-version-of-MapSwipe) + +## Select appropriate Mapillary imagery for the tutorial (with JOSM and Mapillary plug-in) + +1. Open JOSM. Make sure the [JOSM Mapillary plug-in](https://wiki.openstreetmap.org/wiki/JOSM/Plugins/Mapillary) is installed +2. **File > Download data**. Select an area in which you expect appropriate example imagery available on Mapillary and **Download** +3. **Imagery > Mapillary** to download sequences and images for the current area +4. If helpful, use the Mapillary filter dialog to filter images (for start and end date, user and/or organization) +5. Click **Mapillary** in Layers controls to select the Mapillary layer +6. Zoom in until you can see images location markers (green dots) +7. Click on the dots to view the images +8. Once you have found an image that you would like to use in your tutorial, **File > Export Mapillary images** and select **Export selected images** +9. Click **Explore** +10. Choose a parent folder for all images in this tutorial +11. **OK** +12. Repeat until you have exported all the images that you would like to use in the tutorial. Use the same parent folder for all images. + +## Add exported Mapillary images as geotagged images in QGIS + +1. Open QGIS +2. **Processing Toolbox > Vector creation > Import geotagged photos** +3. Select the folder containing all exported Mapillary images and check **Scan recursively** +4. **Run** +5. **Properties > Display** and add `` to HTML Map Tip to show images on a pop up +6. **View > Show Map Tips** +7. If you keep the mouse tip on the image markers, a pop up with the image will appear + +## Edit geotagged images in QGIS + +1. Right click on layer. +2. **Properties > Field** +3. **Toggle editing mode** +4. Change the name of the `filename` column to `id` +5. Add `Integer (32 bit)` columns titled `screen` and `reference`. +6. Populate the `reference` and `screen` fields. + * `reference` is the value of the correct answer option for the image. + * `screen` determines the order of the images in the tutorial and should start with `1`. +7. Delete any rows representing images that you do not want to use + +## Export as GeoJSON + +1. **Toggle editing mode** +2. **Save** +3. Right click, **Export > Save Features As...** +4. Choose Format GeoJSON, CRS EPSG:4326 - WGS 84 +5. Select only `id`, `reference` and `screen` as fields to export. Deselect all other fields. +6. Choose a file name and location and click OK to save + +## Create tutorial + +1. Go to https://dev-managers.mapswipe.org/ +2. Select **Projects** and then **Add New Tutorial**. +3. Check that **Project Type** is set to **Street**. +4. Fill in all the fields, following the instructions. Upload your `GeoJSON` you just created with the scenarios where it says **Scenario Pages**. +5. Submit diff --git a/mapswipe_workers/sample_data/street/street_tutorial_sample_scenario.geojson b/mapswipe_workers/sample_data/street/street_tutorial_sample_scenario.geojson new file mode 100644 index 000000000..8f5b236f0 --- /dev/null +++ b/mapswipe_workers/sample_data/street/street_tutorial_sample_scenario.geojson @@ -0,0 +1,17 @@ +{ + "type": "FeatureCollection", + "name": "cobblestone-scenario", + "crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } }, + "features": [ + { + "type": "Feature", + "properties": { "id": "378811598610667", "reference": 1, "screen": 2 }, + "geometry": { "type": "Point", "coordinates": [ 13.45285, 52.508467, 0.0 ] } + }, + { + "type": "Feature", + "properties": { "id": "1171343450849316", "reference": 0, "screen": 1 }, + "geometry": { "type": "Point", "coordinates": [ 13.4514123, 52.5103378, 0.0 ] } + } + ] +} diff --git a/mapswipe_workers/tests/fixtures/mapillary_response.csv b/mapswipe_workers/tests/fixtures/mapillary_response.csv new file mode 100644 index 000000000..2988a1dc4 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/mapillary_response.csv @@ -0,0 +1,8 @@ +geometry,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id +POINT (38.995129466056824 -6.785243670271996),1453463352000,102506575322825,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ,1 +POINT (38.99839103221893 -6.7866606090858),1679465543298,118200124520512,603013591724120,,False,104.904435758052,Tj9u08PcRnQEAU13yeNhkr +POINT (39.000306129455566 -6.787576822940906),1679465564715,118200124520512,547943597327117,,False,103.540136343809,Tj9u08PcRnQEAU13yeNhkr +POINT (38.9906769990921 -6.783315348346505),1453463400000,102506575322825,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ +POINT (38.99797797203064 -6.786490150501777),1679465534081,118200124520512,3271824289814895,,False,112.637054443359,Tj9u08PcRnQEAU13yeNhkr +POINT (39.00127172470093 -6.787981661065601),1453463294000,102506575322825,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ + diff --git a/mapswipe_workers/tests/fixtures/mapillary_sequence.csv b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv new file mode 100644 index 000000000..597fa6f66 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/mapillary_sequence.csv @@ -0,0 +1,71 @@ +,geometry,captured_at,creator_id,id,image_id,is_pano,compass_angle,sequence_id,organization_id +9,POINT (38.995129466056824 -6.785243670271996),1453463352000.0,102506575322825.0,371897427508205,,True,292.36693993283,ywMkSP_5PaJzcbIDa5v1aQ, +12,POINT (38.9906769990921 -6.783315348346505),1453463400000.0,102506575322825.0,503298877423056,,True,286.12725090647,ywMkSP_5PaJzcbIDa5v1aQ, +14,POINT (39.00127172470093 -6.787981661065601),1453463294000.0,102506575322825.0,2897708763800777,,True,296.09895739855,ywMkSP_5PaJzcbIDa5v1aQ, +18,POINT (38.99769365787506 -6.786351652857817),1453463332000.0,102506575322825.0,1014398349364928,,True,288.49790876724,ywMkSP_5PaJzcbIDa5v1aQ, +19,POINT (38.99540305137634 -6.785360860858361),1453463350000.0,102506575322825.0,165035685525369,,True,293.57188365181,ywMkSP_5PaJzcbIDa5v1aQ, +20,POINT (39.00015592575073 -6.787491593818643),1453463300000.0,102506575322825.0,1139870886511452,,True,305.58569254109,ywMkSP_5PaJzcbIDa5v1aQ, +21,POINT (38.9979350566864 -6.7864262285172),1453463330000.0,102506575322825.0,921248855101166,,True,287.76018227153,ywMkSP_5PaJzcbIDa5v1aQ, +22,POINT (38.99890601634979 -6.786804433469129),1453463318000.0,102506575322825.0,233272625257058,,True,295.57372041131,ywMkSP_5PaJzcbIDa5v1aQ, +23,POINT (38.99618625640869 -6.7857071056060505),1453463344000.0,102506575322825.0,762184237820558,,True,293.88722229187,ywMkSP_5PaJzcbIDa5v1aQ, +26,POINT (38.98969531059265 -6.782905179427416),1453463424000.0,102506575322825.0,1099193250486391,,True,292.87773588969,ywMkSP_5PaJzcbIDa5v1aQ, +27,POINT (38.998627066612244 -6.786671262745287),1453463322000.0,102506575322825.0,831521697449190,,True,296.54786608582,ywMkSP_5PaJzcbIDa5v1aQ, +29,POINT (38.992629647254944 -6.784167646282242),1453463368000.0,102506575322825.0,151617226911336,,True,292.93390738544,ywMkSP_5PaJzcbIDa5v1aQ, +30,POINT (39.00003254413605 -6.7873584232849),1453463302000.0,102506575322825.0,1773698542801178,,True,296.75481871036,ywMkSP_5PaJzcbIDa5v1aQ, +32,POINT (39.000563621520996 -6.787811202949342),1453463298000.0,102506575322825.0,164116422292333,,True,305.80313236719,ywMkSP_5PaJzcbIDa5v1aQ, +37,POINT (38.9902800321579 -6.783144888578377),1453463410000.0,102506575322825.0,329957465177767,,True,293.87481737883,ywMkSP_5PaJzcbIDa5v1aQ, +38,POINT (38.99117052555084 -6.783539076700606),1453463388000.0,102506575322825.0,2948533428757015,,True,294.44274652914,ywMkSP_5PaJzcbIDa5v1aQ, +39,POINT (38.99877190589905 -6.78674051152629),1453463320000.0,102506575322825.0,479325670049740,,True,296.24340777377,ywMkSP_5PaJzcbIDa5v1aQ, +41,POINT (38.990601897239685 -6.783294040878786),1453463402000.0,102506575322825.0,501560441022559,,True,289.98678899102,ywMkSP_5PaJzcbIDa5v1aQ, +42,POINT (38.989362716674805 -6.7828785450699485),1453463432000.0,102506575322825.0,494436578424418,,True,249.25945175736,ywMkSP_5PaJzcbIDa5v1aQ, +44,POINT (38.994566202163696 -6.7850039621655895),1453463356000.0,102506575322825.0,2928848347373461,,True,295.93075138027,ywMkSP_5PaJzcbIDa5v1aQ, +45,POINT (38.993815183639526 -6.784657716912335),1453463362000.0,102506575322825.0,167884815220625,,True,290.65289338004,ywMkSP_5PaJzcbIDa5v1aQ, +47,POINT (38.991841077804565 -6.783837381011111),1453463380000.0,102506575322825.0,2783373888590755,,True,292.94668882511,ywMkSP_5PaJzcbIDa5v1aQ, +48,POINT (38.99052679538727 -6.783262079675453),1453463404000.0,102506575322825.0,500930794384261,,True,296.09431621523,ywMkSP_5PaJzcbIDa5v1aQ, +49,POINT (38.9897757768631 -6.782937140654425),1453463422000.0,102506575322825.0,473363863989539,,True,292.22088072734,ywMkSP_5PaJzcbIDa5v1aQ, +50,POINT (38.99429798126221 -6.784870790943785),1453463358000.0,102506575322825.0,792308461667709,,True,295.36479453372,ywMkSP_5PaJzcbIDa5v1aQ, +51,POINT (38.9997535943985 -6.787219925890724),1453463306000.0,102506575322825.0,1169832606865116,,True,296.51874301031,ywMkSP_5PaJzcbIDa5v1aQ, +54,POINT (38.992860317230225 -6.784268856561923),1453463364000.0,102506575322825.0,143904254368287,,True,292.87299883627,ywMkSP_5PaJzcbIDa5v1aQ, +57,POINT (38.98994743824005 -6.783006389972371),1453463418000.0,102506575322825.0,512708183243254,,True,292.58758044439,ywMkSP_5PaJzcbIDa5v1aQ, +58,POINT (38.99670124053955 -6.785941486524692),1453463340000.0,102506575322825.0,168474601828325,,True,294.32908734047,ywMkSP_5PaJzcbIDa5v1aQ, +59,POINT (38.992136120796204 -6.783959898799395),1453463376000.0,102506575322825.0,171815874817246,,True,292.47687051975,ywMkSP_5PaJzcbIDa5v1aQ, +60,POINT (38.99090766906738 -6.783405905073778),1453463394000.0,102506575322825.0,475809606904698,,True,297.28158189053,ywMkSP_5PaJzcbIDa5v1aQ, +61,POINT (38.99251699447632 -6.7841250314212544),1453463370000.0,102506575322825.0,798930200741228,,True,292.63573224675,ywMkSP_5PaJzcbIDa5v1aQ, +62,POINT (38.99019956588745 -6.7831129273651385),1453463412000.0,102506575322825.0,989719805170705,,True,292.67075887406,ywMkSP_5PaJzcbIDa5v1aQ, +63,POINT (38.991336822509766 -6.783613652795566),1453463386000.0,102506575322825.0,887351401825944,,True,294.11111203184,ywMkSP_5PaJzcbIDa5v1aQ, +64,POINT (38.99745762348175 -6.786271750352796),1453463334000.0,102506575322825.0,820185135568044,,True,292.38261450394,ywMkSP_5PaJzcbIDa5v1aQ, +69,POINT (38.99919033050537 -6.7869376041561225),1453463314000.0,102506575322825.0,1401323273568889,,True,296.07037190303,ywMkSP_5PaJzcbIDa5v1aQ, +70,POINT (38.99229168891907 -6.784023821111347),1453463374000.0,102506575322825.0,971311816939999,,True,293.93020293096,ywMkSP_5PaJzcbIDa5v1aQ, +71,POINT (38.9956659078598 -6.785483378259087),1453463348000.0,102506575322825.0,1888667171315269,,True,294.14026651816,ywMkSP_5PaJzcbIDa5v1aQ, +77,POINT (38.99095058441162 -6.783437866267576),1453463392000.0,102506575322825.0,313013763568992,,True,299.51413646126,ywMkSP_5PaJzcbIDa5v1aQ, +78,POINT (38.99240434169769 -6.784077089698172),1453463372000.0,102506575322825.0,300316491494684,,True,293.91812405268,ywMkSP_5PaJzcbIDa5v1aQ, +79,POINT (38.994094133377075 -6.784774907641307),1453463360000.0,102506575322825.0,799986720927690,,True,293.50534124709,ywMkSP_5PaJzcbIDa5v1aQ, +80,POINT (38.998122811317444 -6.786495477333432),1453463328000.0,102506575322825.0,766866250678914,,True,291.66896324881,ywMkSP_5PaJzcbIDa5v1aQ, +82,POINT (38.99715185165405 -6.786143906317193),1453463336000.0,102506575322825.0,4104687166260249,,True,294.20932475917,ywMkSP_5PaJzcbIDa5v1aQ, +84,POINT (38.99081647396088 -6.783368617011661),1453463396000.0,102506575322825.0,311476673697450,,True,292.1387426333,ywMkSP_5PaJzcbIDa5v1aQ, +85,POINT (38.999614119529724 -6.787150677178687),1453463308000.0,102506575322825.0,486545769438431,,True,296.75245798459,ywMkSP_5PaJzcbIDa5v1aQ, +86,POINT (38.99830520153046 -6.786516784659511),1453463326000.0,102506575322825.0,287818266220456,,True,296.90157475174,ywMkSP_5PaJzcbIDa5v1aQ, +89,POINT (38.991073966026306 -6.78349646178404),1453463390000.0,102506575322825.0,1104648166685558,,True,294.64381621501,ywMkSP_5PaJzcbIDa5v1aQ, +95,POINT (38.99045169353485 -6.783224791602194),1453463406000.0,102506575322825.0,1189096484862313,,True,296.52075233925,ywMkSP_5PaJzcbIDa5v1aQ, +96,POINT (38.989604115486145 -6.782867891326546),1453463426000.0,102506575322825.0,149492403718498,,True,291.35167681135,ywMkSP_5PaJzcbIDa5v1aQ, +99,POINT (38.990371227264404 -6.783187503526065),1453463408000.0,102506575322825.0,1891398304374063,,True,293.92928705821,ywMkSP_5PaJzcbIDa5v1aQ, +102,POINT (38.99695873260498 -6.786058676941238),1453463338000.0,102506575322825.0,823059255257426,,True,294.12835748694,ywMkSP_5PaJzcbIDa5v1aQ, +105,POINT (38.99198055267334 -6.7838959764789735),1453463378000.0,102506575322825.0,3696263247264941,,True,293.15037639266,ywMkSP_5PaJzcbIDa5v1aQ, +108,POINT (38.989861607551575 -6.782974428749938),1453463420000.0,102506575322825.0,862036401042035,,True,291.44770311405,ywMkSP_5PaJzcbIDa5v1aQ, +109,POINT (38.98951292037964 -6.782841256967004),1453463428000.0,102506575322825.0,304865891166877,,True,272.6464361279,ywMkSP_5PaJzcbIDa5v1aQ, +111,POINT (38.99274230003357 -6.784220914853137),1453463366000.0,102506575322825.0,794241458144562,,True,293.2506353664,ywMkSP_5PaJzcbIDa5v1aQ, +113,POINT (38.99847149848938 -6.786596687123861),1453463324000.0,102506575322825.0,939866586773160,,True,295.61868532377,ywMkSP_5PaJzcbIDa5v1aQ, +115,POINT (38.99947464466095 -6.787081428456702),1453463310000.0,102506575322825.0,1143620816058871,,True,296.46351643457,ywMkSP_5PaJzcbIDa5v1aQ, +116,POINT (38.99003326892853 -6.783043678062526),1453463416000.0,102506575322825.0,1405161323178994,,True,293.46012249789,ywMkSP_5PaJzcbIDa5v1aQ, +119,POINT (39.00104105472565 -6.78785914430064),1453463296000.0,102506575322825.0,1273957886333326,,True,279.89824953791,ywMkSP_5PaJzcbIDa5v1aQ, +120,POINT (38.98943245410919 -6.782857237582903),1453463430000.0,102506575322825.0,1171907646606759,,True,252.08147683237,ywMkSP_5PaJzcbIDa5v1aQ, +121,POINT (38.99148166179657 -6.783677575153462),1453463384000.0,102506575322825.0,164230852280713,,True,293.89449177018,ywMkSP_5PaJzcbIDa5v1aQ, +122,POINT (38.999335169792175 -6.787012179724755),1453463312000.0,102506575322825.0,510570906624460,,True,297.86598575521,ywMkSP_5PaJzcbIDa5v1aQ, +124,POINT (38.99592339992523 -6.785595241945558),1453463346000.0,102506575322825.0,136548521821574,,True,293.97664288588,ywMkSP_5PaJzcbIDa5v1aQ, +125,POINT (38.990119099617004 -6.783075639280355),1453463414000.0,102506575322825.0,1112069379291585,,True,293.19285658924,ywMkSP_5PaJzcbIDa5v1aQ, +126,POINT (38.99644374847412 -6.785829622918669),1453463342000.0,102506575322825.0,846432512639247,,True,294.78265773594,ywMkSP_5PaJzcbIDa5v1aQ, +128,POINT (38.99165332317352 -6.783752151226963),1453463382000.0,102506575322825.0,133891088754131,,True,294.07658010782,ywMkSP_5PaJzcbIDa5v1aQ, +129,POINT (38.99078965187073 -6.7833579632791015),1453463398000.0,102506575322825.0,211793933759059,,True,294.17362400652,ywMkSP_5PaJzcbIDa5v1aQ, +131,POINT (38.99904549121857 -6.786873682230961),1453463316000.0,102506575322825.0,137437058395340,,True,295.32783278431,ywMkSP_5PaJzcbIDa5v1aQ, +132,POINT (38.99989306926727 -6.787289174592786),1453463304000.0,102506575322825.0,323401522460093,,True,296.4023335068,ywMkSP_5PaJzcbIDa5v1aQ, +133,POINT (38.994882702827454 -6.785147787043741),1453463354000.0,102506575322825.0,1070660863461711,,True,294.24600460049,ywMkSP_5PaJzcbIDa5v1aQ, diff --git a/mapswipe_workers/tests/fixtures/projectDrafts/street.json b/mapswipe_workers/tests/fixtures/projectDrafts/street.json new file mode 100644 index 000000000..67d1d8b04 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/projectDrafts/street.json @@ -0,0 +1,50 @@ +{ + "createdBy": "test", + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 39.27186980415655, + -6.818313681620424 + ], + [ + 39.27186980415655, + -6.824056026803248 + ], + [ + 39.27489288297136, + -6.823996705403303 + ], + [ + 39.27483313833096, + -6.817969613314901 + ], + [ + 39.27186980415655, + -6.818313681620424 + ] + ] + ] + } + } + ] + }, + "image": "", + "lookFor": "buildings", + "name": "test - Dar es Salaam (1)\ntest", + "projectDetails": "test", + "projectNumber": 1, + "projectTopic": "test", + "projectType": 7, + "requestingOrganisation": "test", + "verificationNumber": 3, + "groupSize": 25, + "samplingThreshold": 0.1 +} diff --git a/mapswipe_workers/tests/fixtures/tutorialDrafts/change_detection.json b/mapswipe_workers/tests/fixtures/tutorialDrafts/change_detection.json index 4b857eaa0..8c0f817bd 100644 --- a/mapswipe_workers/tests/fixtures/tutorialDrafts/change_detection.json +++ b/mapswipe_workers/tests/fixtures/tutorialDrafts/change_detection.json @@ -4,6 +4,7 @@ "exampleImage2": "", "lookFor": "damaged buildings", "name": "change_detection_tutorial", + "tutorialDraftId": "test_tile_change_detection", "projectType": 3, "screens": [ null, diff --git a/mapswipe_workers/tests/fixtures/tutorialDrafts/completeness.json b/mapswipe_workers/tests/fixtures/tutorialDrafts/completeness.json index b08c10dd7..0752c71a1 100644 --- a/mapswipe_workers/tests/fixtures/tutorialDrafts/completeness.json +++ b/mapswipe_workers/tests/fixtures/tutorialDrafts/completeness.json @@ -4,6 +4,7 @@ "exampleImage2": "https://firebasestorage.googleapis.com/v0/b/heigit-crowdmap.appspot.com/o/projectImages%2F1686065132355-tutorial-image-2-1x1.png?alt=media&token=bf8e67bc-d34c-4676-ba17-56bffc6b3f2d", "lookFor": "buildings", "name": "completeness_tutorial", + "tutorialDraftId": "test_tile_completeness", "projectType": 4, "screens": { "categories": { diff --git a/mapswipe_workers/tests/fixtures/tutorialDrafts/footprint.json b/mapswipe_workers/tests/fixtures/tutorialDrafts/footprint.json index b8b31a9f9..b4e26e7bd 100644 --- a/mapswipe_workers/tests/fixtures/tutorialDrafts/footprint.json +++ b/mapswipe_workers/tests/fixtures/tutorialDrafts/footprint.json @@ -1,5 +1,6 @@ { "createdBy": "LtCUyou6CnSSc1H0Q0nDrN97x892", + "tutorialDraftId": "test_footprint_tutorial", "customOptions": [ { "description": "the shape does outline a building in the image", diff --git a/mapswipe_workers/tests/fixtures/tutorialDrafts/street.json b/mapswipe_workers/tests/fixtures/tutorialDrafts/street.json new file mode 100644 index 000000000..1385ffb52 --- /dev/null +++ b/mapswipe_workers/tests/fixtures/tutorialDrafts/street.json @@ -0,0 +1,110 @@ +{ + "createdBy": "atCSosZACaN0qhcVjtMO1tq9d1G3", + "tutorialDraftId": "test_tile_classification", + "informationPages": [ + { + "blocks": [ + { + "blockNumber": 1, + "blockType": "text", + "textDescription": "This is the first information page" + }, + { + "blockNumber": 2, + "blockType": "image", + "image": "https://firebasestorage.googleapis.com/v0/b/dev-mapswipe.appspot.com/o/tutorialImages%2F1739963139725-block-image-2-1x1.png?alt=media&token=ae584dcd-d351-4bfe-be5f-1e0d38547f72" + } + ], + "pageNumber": 1, + "title": "Information page 1" + } + ], + "lookFor": "cobblestone", + "name": "cobblestone-tutorial", + "projectType": 7, + "screens": [ + null, + { + "hint": { + "description": "This seems to be a tarmac surface.", + "icon": "check", + "title": "Tarmac" + }, + "instructions": { + "description": "Check out if the road surface material is cobblestone here", + "icon": "check", + "title": "Is this cobblestone?" + }, + "success": { + "description": "Correct, this is not cobblestone", + "icon": "check", + "title": "Nice!" + } + }, + { + "hint": { + "description": "That surface does look like cobblestone!", + "icon": "heart-outline", + "title": "Cobblestone" + }, + "instructions": { + "description": "Does this look like cobblestone?", + "icon": "egg-outline", + "title": "How about this one?" + }, + "success": { + "description": "Correct", + "icon": "search-outline", + "title": "Correct" + } + } + ], + "tileServer": { + "credits": "© 2019 Microsoft Corporation, Earthstar Geographics SIO", + "name": "bing" + }, + "tutorialTasks": { + "crs": { + "properties": { + "name": "urn:ogc:def:crs:OGC:1.3:CRS84" + }, + "type": "name" + }, + "features": [ + { + "geometry": { + "coordinates": [ + 13.4514123, + 52.5103378, + 0 + ], + "type": "Point" + }, + "properties": { + "id": "1171343450849316", + "reference": 1, + "screen": 1 + }, + "type": "Feature" + }, + { + "geometry": { + "coordinates": [ + 13.45285, + 52.508467, + 0 + ], + "type": "Point" + }, + "properties": { + "id": "378811598610667", + "reference": 0, + "screen": 2 + }, + "type": "Feature" + } + ], + "name": "cobblestone-scenario", + "type": "FeatureCollection" + } +} \ No newline at end of file diff --git a/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json new file mode 100644 index 000000000..f945f1661 --- /dev/null +++ b/mapswipe_workers/tests/integration/fixtures/street/projectDrafts/street.json @@ -0,0 +1,50 @@ +{ + "createdBy": "test", + "geometry": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + 39.27186980415655, + -6.818313681620424 + ], + [ + 39.27186980415655, + -6.824056026803248 + ], + [ + 39.27489288297136, + -6.823996705403303 + ], + [ + 39.27483313833096, + -6.817969613314901 + ], + [ + 39.27186980415655, + -6.818313681620424 + ] + ] + ] + } + } + ] + }, + "image": "", + "lookFor": "buildings", + "name": "test - Dar es Salaam (1)\ntest", + "projectDetails": "test", + "projectNumber": 1, + "projectTopic": "test", + "projectType": 7, + "requestingOrganisation": "test", + "verificationNumber": 3, + "groupSize": 25, + "customOptions": [{ "color": "", "label": "", "value": -999 }, { "color": "#008000", "label": "yes", "value": 1 }, { "color": "#FF0000", "label": "no", "value": 2 }, { "color": "#FFA500", "label": "maybe", "value": 3 }] +} diff --git a/mapswipe_workers/tests/integration/set_up.py b/mapswipe_workers/tests/integration/set_up.py index 74adc6fda..1c3c0bdf4 100644 --- a/mapswipe_workers/tests/integration/set_up.py +++ b/mapswipe_workers/tests/integration/set_up.py @@ -16,20 +16,28 @@ def set_firebase_test_data( - project_type: str, data_type: str, fixture_name: str, identifier: str + project_type: str, + data_type: str, + fixture_name: str, + identifier: str, + tutorial_id: str = None, ): test_dir = os.path.dirname(__file__) fixture_name = fixture_name + ".json" file_path = os.path.join( test_dir, "fixtures", project_type, data_type, fixture_name ) - upload_file_to_firebase(file_path, data_type, identifier) + upload_file_to_firebase(file_path, data_type, identifier, tutorial_id=tutorial_id) -def upload_file_to_firebase(file_path: str, data_type: str, identifier: str): +def upload_file_to_firebase( + file_path: str, data_type: str, identifier: str, tutorial_id: str = None +): with open(file_path) as test_file: test_data = json.load(test_file) + if tutorial_id: + test_data["tutorialId"] = tutorial_id fb_db = auth.firebaseDB() ref = fb_db.reference(f"/v2/{data_type}/{identifier}") ref.set(test_data) @@ -85,15 +93,20 @@ def create_test_project( set_postgres_test_data(project_type, "users", "user") set_firebase_test_data(project_type, "user_groups", "user_group", "") set_firebase_test_data(project_type, "results", fixture_name, project_id) - set_postgres_test_data(project_type, "mapping_sessions", fixture_name, columns=[ - "project_id", - "group_id", - "user_id", - "mapping_session_id", - "start_time", - "end_time", - "items_count", - ]) + set_postgres_test_data( + project_type, + "mapping_sessions", + fixture_name, + columns=[ + "project_id", + "group_id", + "user_id", + "mapping_session_id", + "start_time", + "end_time", + "items_count", + ], + ) set_postgres_test_data(project_type, mapping_sessions_results, fixture_name) if create_user_group_session_data: set_postgres_test_data( @@ -108,7 +121,9 @@ def create_test_project( "created_at", ], ) - set_postgres_test_data(project_type, "mapping_sessions_user_groups", fixture_name) + set_postgres_test_data( + project_type, "mapping_sessions_user_groups", fixture_name + ) time.sleep(5) # Wait for Firebase Functions to complete return project_id @@ -131,12 +146,24 @@ def create_test_user(project_type: str, user_id: str = None) -> str: def create_test_project_draft( - project_type: str, fixture_name: str = "user", identifier: str = "" + project_type: str, + fixture_name: str = "user", + identifier: str = "", + tutorial_id: str = None, ) -> str: """ Create test project drafts in Firebase and return project ids. Project drafts in Firebase are created by project manager using the dashboard. """ + if tutorial_id: + set_firebase_test_data( + project_type, + "projectDrafts", + fixture_name, + identifier, + tutorial_id=tutorial_id, + ) + return identifier if not identifier: identifier = f"test_{fixture_name}" set_firebase_test_data(project_type, "projectDrafts", fixture_name, identifier) diff --git a/mapswipe_workers/tests/integration/set_up_db.sql b/mapswipe_workers/tests/integration/set_up_db.sql index ce9b97197..f954d3a8c 100644 --- a/mapswipe_workers/tests/integration/set_up_db.sql +++ b/mapswipe_workers/tests/integration/set_up_db.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(MULTIPOLYGON, 4326), + geom geometry(Geometry, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), diff --git a/mapswipe_workers/tests/integration/tear_down.py b/mapswipe_workers/tests/integration/tear_down.py index 61760781c..33af5d52d 100644 --- a/mapswipe_workers/tests/integration/tear_down.py +++ b/mapswipe_workers/tests/integration/tear_down.py @@ -8,7 +8,7 @@ from mapswipe_workers import auth -def delete_test_data(project_id: str) -> None: +def delete_test_data(project_id: str, tutorial_id: str = None) -> None: """ Delete test project indluding groups, tasks and results from Firebase and Postgres @@ -38,6 +38,12 @@ def delete_test_data(project_id: str) -> None: ref = fb_db.reference(f"v2/users/{project_id}") ref.delete() + if tutorial_id is not None: + ref = fb_db.reference(f"v2/projects/{tutorial_id}") + ref.delete() + ref = fb_db.reference(f"v2/tutorialDrafts/{tutorial_id}") + ref.delete() + # Clear out the user-group used in test. # XXX: Use a firebase simulator for running test. # For CI/CD, use a real firebase with scope using commit hash, diff --git a/mapswipe_workers/tests/integration/test_create_street_project.py b/mapswipe_workers/tests/integration/test_create_street_project.py new file mode 100644 index 000000000..fd0608f98 --- /dev/null +++ b/mapswipe_workers/tests/integration/test_create_street_project.py @@ -0,0 +1,61 @@ +import unittest + +from click.testing import CliRunner + +from mapswipe_workers import auth, mapswipe_workers +from mapswipe_workers.definitions import logger +from mapswipe_workers.utils.create_directories import create_directories +from tests.integration import set_up, tear_down + + +class TestCreateStreetProject(unittest.TestCase): + def setUp(self): + self.project_id = [ + set_up.create_test_project_draft("street", "street"), + ] + create_directories() + + def tearDown(self): + for element in self.project_id: + tear_down.delete_test_data(element) + + def test_create_street_project(self): + runner = CliRunner() + result = runner.invoke( + mapswipe_workers.run_create_projects, catch_exceptions=False + ) + if result.exit_code != 0: + raise result.exception + pg_db = auth.postgresDB() + for element in self.project_id: + logger.info(f"Checking project {self.project_id}") + query = "SELECT project_id FROM projects WHERE project_id = %s" + result = pg_db.retr_query(query, [element])[0][0] + self.assertEqual(result, element) + + # check if tasks made it to postgres + query = """ + SELECT count(*) + FROM tasks + WHERE project_id = %s + """ + result = pg_db.retr_query(query, [element])[0][0] + self.assertGreater(result, 0) + + fb_db = auth.firebaseDB() + ref = fb_db.reference(f"/v2/projects/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + ref = fb_db.reference(f"/v2/groups/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + # Street projects have tasks in Firebase + ref = fb_db.reference(f"/v2/tasks/{element}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/integration/test_create_tutorial.py b/mapswipe_workers/tests/integration/test_create_tutorial.py new file mode 100644 index 000000000..e6db39579 --- /dev/null +++ b/mapswipe_workers/tests/integration/test_create_tutorial.py @@ -0,0 +1,45 @@ +import unittest + +from click.testing import CliRunner + +from mapswipe_workers import auth, mapswipe_workers +from mapswipe_workers.utils.create_directories import create_directories +from tests.integration import set_up, tear_down + + +class TestCreateTileClassificationProject(unittest.TestCase): + def setUp(self): + self.tutorial_id = set_up.create_test_tutorial_draft( + "tile_classification", + "tile_classification", + "test_tile_classification_tutorial", + ) + + self.project_id = set_up.create_test_project_draft( + "tile_classification", + "tile_classification", + "test_tile_classification_tutorial", + tutorial_id=self.tutorial_id, + ) + create_directories() + + def tearDown(self): + tear_down.delete_test_data(self.project_id, self.tutorial_id) + + def test_create_tile_classification_project_and_tutorial(self): + runner = CliRunner() + runner.invoke(mapswipe_workers.run_create_tutorials, catch_exceptions=False) + runner.invoke(mapswipe_workers.run_create_projects, catch_exceptions=False) + + fb_db = auth.firebaseDB() + ref = fb_db.reference(f"/v2/projects/{self.project_id}") + result = ref.get() + self.assertEqual(result["tutorialId"], self.tutorial_id) + + ref = fb_db.reference(f"/v2/projects/{self.tutorial_id}") + result = ref.get(shallow=True) + self.assertIsNotNone(result) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_process_mapillary.py b/mapswipe_workers/tests/unittests/test_process_mapillary.py new file mode 100644 index 000000000..341c6a601 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_process_mapillary.py @@ -0,0 +1,350 @@ +import json +import os +import unittest +from unittest.mock import MagicMock, patch + +import pandas as pd +from shapely import wkt +from shapely.geometry import GeometryCollection, MultiPolygon, Point, Polygon + +from mapswipe_workers.definitions import CustomError +from mapswipe_workers.utils.process_mapillary import ( + coordinate_download, + create_tiles, + download_and_process_tile, + filter_by_timerange, + filter_results, + geojson_to_polygon, + get_image_metadata, +) + + +class TestTileGroupingFunctions(unittest.TestCase): + @classmethod + def setUpClass(cls): + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "feature_collection.json", + ), + "r", + ) as file: + cls.fixture_data = json.load(file) + + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_response.csv", + ), + "r", + ) as file: + df = pd.read_csv(file) + df["geometry"] = df["geometry"].apply(wkt.loads) + cls.fixture_df = df + + def setUp(self): + self.level = 14 + self.test_polygon = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)]) + self.empty_polygon = Polygon() + self.empty_geometry = GeometryCollection() + self.row = pd.Series({"x": 1, "y": 1, "z": self.level}) + + def test_create_tiles_with_valid_polygon(self): + tiles = create_tiles(self.test_polygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertFalse(tiles.empty) + + def test_create_tiles_with_multipolygon(self): + polygon = Polygon( + [ + (0.00000000, 0.00000000), + (0.000000001, 0.00000000), + (0.00000000, 0.000000001), + (0.00000000, 0.000000001), + ] + ) + multipolygon = MultiPolygon([polygon, polygon]) + tiles = create_tiles(multipolygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertFalse(tiles.empty) + self.assertEqual(len(tiles), 1) + + def test_create_tiles_with_empty_polygon(self): + tiles = create_tiles(self.empty_polygon, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertTrue(tiles.empty) + + def test_create_tiles_with_empty_geometry(self): + tiles = create_tiles(self.empty_geometry, self.level) + self.assertIsInstance(tiles, pd.DataFrame) + self.assertTrue(tiles.empty) + + def test_geojson_to_polygon_feature_collection_with_multiple_polygons(self): + geojson_data = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + }, + }, + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]], + }, + }, + ], + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, MultiPolygon) + self.assertEqual(len(result.geoms), 2) + + def test_geojson_to_polygon_single_feature_polygon(self): + geojson_data = { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + }, + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, Polygon) + + def test_geojson_to_polygon_single_feature_multipolygon(self): + geojson_data = { + "type": "Feature", + "geometry": { + "type": "MultiPolygon", + "coordinates": [ + [[(0, 0), (1, 0), (1, 1), (0, 1), (0, 0)]], + [[(2, 2), (3, 2), (3, 3), (2, 3), (2, 2)]], + ], + }, + } + result = geojson_to_polygon(geojson_data) + self.assertIsInstance(result, MultiPolygon) + self.assertEqual(len(result.geoms), 2) + + def test_geojson_to_polygon_non_polygon_geometry_in_feature_collection(self): + geojson_data = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "LineString", "coordinates": [(0, 0), (1, 1)]}, + } + ], + } + with self.assertRaises(ValueError) as context: + geojson_to_polygon(geojson_data) + self.assertEqual( + str(context.exception), + "Non-polygon geometries cannot be combined into a MultiPolygon.", + ) + + def test_geojson_to_polygon_empty_feature_collection(self): + geojson_data = {"type": "FeatureCollection", "features": []} + result = geojson_to_polygon(geojson_data) + self.assertTrue(result.is_empty) + + def test_geojson_to_polygon_contribution_geojson(self): + result = geojson_to_polygon(self.fixture_data) + self.assertIsInstance(result, Polygon) + + @patch( + "mapswipe_workers.utils.process_mapillary.vt2geojson_tools.vt_bytes_to_geojson" + ) + @patch("mapswipe_workers.utils.process_mapillary.requests.get") + def test_download_and_process_tile_success(self, mock_get, mock_vt2geojson): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.content = b"mock vector tile data" # Example mock data + mock_get.return_value = mock_response + + mock_vt2geojson.return_value = { + "features": [ + { + "geometry": {"type": "Point", "coordinates": [0, 0]}, + "properties": {"id": 1}, + } + ] + } + + row = {"x": 1, "y": 1, "z": 14} + + polygon = wkt.loads("POLYGON ((-1 -1, -1 1, 1 1, 1 -1, -1 -1))") + + result = download_and_process_tile(row, polygon, {}) + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 1) + self.assertEqual(result["geometry"][0].wkt, "POINT (0 0)") + + @patch("mapswipe_workers.utils.process_mapillary.requests.get") + def test_download_and_process_tile_failure(self, mock_get): + + mock_response = MagicMock() + mock_response.status_code = 500 + mock_get.return_value = mock_response + + result = download_and_process_tile(self.row, self.test_polygon, {}) + + self.assertIsNone(result) + + @patch("mapswipe_workers.utils.process_mapillary.get_mapillary_data") + def test_download_and_process_tile_spatial_filtering(self, mock_get_mapillary_data): + inside_points = [ + (0.2, 0.2), + (0.5, 0.5), + ] + outside_points = [ + (1.5, 0.5), + (0.5, 1.5), + (-0.5, 0.5), + ] + points = inside_points + outside_points + data = [ + { + "geometry": Point(x, y), + } + for x, y in points + ] + + mock_get_mapillary_data.return_value = pd.DataFrame(data) + + metadata = download_and_process_tile(self.row, self.test_polygon, {}) + + metadata = metadata.drop_duplicates() + self.assertEqual(len(metadata), len(inside_points)) + + self.assertIsInstance(metadata, pd.DataFrame) + + @patch("mapswipe_workers.utils.process_mapillary.parallelized_processing") + def test_coordinate_download_with_failures(self, mock_parallelized_processing): + mock_parallelized_processing.return_value = pd.DataFrame() + + metadata = coordinate_download(self.test_polygon, self.level, {}) + + self.assertTrue(metadata.empty) + + def test_filter_within_time_range(self): + start_time = "2016-01-20 00:00:00" + end_time = "2022-01-21 23:59:59" + filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) + + self.assertEqual(len(filtered_df), 3) + self.assertTrue(all(filtered_df["captured_at"] >= pd.to_datetime(start_time))) + self.assertTrue(all(filtered_df["captured_at"] <= pd.to_datetime(end_time))) + + def test_filter_without_end_time(self): + start_time = "2020-01-20 00:00:00" + filtered_df = filter_by_timerange(self.fixture_df, start_time) + + self.assertEqual(len(filtered_df), 3) + self.assertTrue(all(filtered_df["captured_at"] >= pd.to_datetime(start_time))) + + def test_filter_time_no_data(self): + start_time = "2016-01-30 00:00:00" + end_time = "2016-01-31 00:00:00" + filtered_df = filter_by_timerange(self.fixture_df, start_time, end_time) + self.assertTrue(filtered_df.empty) + + def test_filter_default(self): + filtered_df = filter_results(self.fixture_df) + self.assertTrue(len(filtered_df) == len(self.fixture_df)) + + def test_filter_pano_true(self): + filtered_df = filter_results(self.fixture_df, is_pano=True) + self.assertEqual(len(filtered_df), 3) + + def test_filter_pano_false(self): + filtered_df = filter_results(self.fixture_df, is_pano=False) + self.assertEqual(len(filtered_df), 3) + + def test_filter_organization_id(self): + filtered_df = filter_results(self.fixture_df, organization_id=1) + self.assertEqual(len(filtered_df), 1) + + def test_filter_creator_id(self): + filtered_df = filter_results(self.fixture_df, creator_id=102506575322825) + self.assertEqual(len(filtered_df), 3) + + def test_filter_time_range(self): + start_time = "2016-01-20 00:00:00" + end_time = "2022-01-21 23:59:59" + filtered_df = filter_results( + self.fixture_df, start_time=start_time, end_time=end_time + ) + self.assertEqual(len(filtered_df), 3) + + def test_filter_no_rows_after_filter(self): + filtered_df = filter_results(self.fixture_df, is_pano="False") + self.assertTrue(filtered_df.empty) + + def test_filter_missing_columns(self): + columns_to_check = [ + "is_pano", + "organization_id", + "captured_at", + ] + for column in columns_to_check: + df_copy = self.fixture_df.copy() + df_copy[column] = None + if column == "captured_at": + column = "start_time" + + result = filter_results(df_copy, **{column: True}) + self.assertIsNone(result) + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata(self, mock_coordinate_download): + mock_coordinate_download.return_value = self.fixture_df + result = get_image_metadata(self.fixture_data) + self.assertIsInstance(result, dict) + self.assertIn("ids", result) + self.assertIn("geometries", result) + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_empty_response(self, mock_coordinate_download): + df = self.fixture_df.copy() + df = df.drop(df.index) + mock_coordinate_download.return_value = df + + with self.assertRaises(CustomError): + get_image_metadata(self.fixture_data) + + @patch("mapswipe_workers.utils.process_mapillary.filter_results") + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_size_restriction( + self, mock_coordinate_download, mock_filter_results + ): + mock_df = pd.DataFrame({"id": range(1, 100002), "geometry": range(1, 100002)}) + mock_coordinate_download.return_value = mock_df + with self.assertRaises(CustomError): + get_image_metadata(self.fixture_data) + + @patch("mapswipe_workers.utils.process_mapillary.coordinate_download") + def test_get_image_metadata_drop_duplicates(self, mock_coordinate_download): + test_df = pd.DataFrame( + { + "id": [1, 2, 2, 3, 4, 4, 5], + "geometry": ["a", "b", "b", "c", "d", "d", "e"], + } + ) + mock_coordinate_download.return_value = test_df + return_dict = get_image_metadata(self.fixture_data) + + return_df = pd.DataFrame(return_dict) + + self.assertNotEqual(len(return_df), len(test_df)) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_project_type_street.py b/mapswipe_workers/tests/unittests/test_project_type_street.py new file mode 100644 index 000000000..8ec8d0fa4 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_project_type_street.py @@ -0,0 +1,55 @@ +import json +import os +import unittest +from unittest.mock import patch +from shapely import wkt +import pandas as pd + +from mapswipe_workers.project_types import StreetProject +from tests import fixtures + + +class TestCreateStreetProject(unittest.TestCase): + def setUp(self) -> None: + project_draft = fixtures.get_fixture( + os.path.join( + "projectDrafts", + "street.json", + ) + ) + project_draft["projectDraftId"] = "foo" + + with patch( + "mapswipe_workers.utils.process_mapillary.coordinate_download" + ) as mock_get: + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_response.csv", + ), + "r", + ) as file: + df = pd.read_csv(file) + df['geometry'] = df['geometry'].apply(wkt.loads) + + mock_get.return_value = df + self.project = StreetProject(project_draft) + + def test_init(self): + self.assertEqual(self.project.geometry["type"], "FeatureCollection") + + def test_create_group(self): + self.project.create_groups() + self.assertTrue(self.project.groups) + + def test_create_tasks(self): + imageId = self.project.imageIds[-1] + self.project.create_groups() + self.project.create_tasks() + self.assertEqual(self.project.tasks["g0"][0].taskId, imageId) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_spatial_sampling.py b/mapswipe_workers/tests/unittests/test_spatial_sampling.py new file mode 100644 index 000000000..8c5aea418 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_spatial_sampling.py @@ -0,0 +1,103 @@ +import os +import unittest + +import numpy as np +import pandas as pd +from shapely import wkt +from shapely.geometry import Point + +from mapswipe_workers.utils.spatial_sampling import ( + distance_on_sphere, + filter_points, + spatial_sampling, +) + + +class TestDistanceCalculations(unittest.TestCase): + @classmethod + def setUpClass(cls): + with open( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "fixtures", + "mapillary_sequence.csv", + ), + "r", + ) as file: + df = pd.read_csv(file) + df["geometry"] = df["geometry"].apply(wkt.loads) + + cls.fixture_df = df + + def test_distance_on_sphere(self): + p1 = Point(-74.006, 40.7128) + p2 = Point(-118.2437, 34.0522) + + distance = distance_on_sphere((p1.x, p1.y), (p2.x, p2.y)) + expected_distance = 3940 # Approximate known distance in km + + self.assertTrue(np.isclose(distance, expected_distance, atol=50)) + + def test_filter_points(self): + data = { + "geometry": [ + "POINT (-74.006 40.7128)", + "POINT (-75.006 41.7128)", + "POINT (-76.006 42.7128)", + "POINT (-77.006 43.7128)", + ] + } + df = pd.DataFrame(data) + + df["geometry"] = df["geometry"].apply(wkt.loads) + + df["long"] = df["geometry"].apply( + lambda geom: geom.x if geom.geom_type == "Point" else None + ) + df["lat"] = df["geometry"].apply( + lambda geom: geom.y if geom.geom_type == "Point" else None + ) + threshold_distance = 100 + filtered_df = filter_points(df, threshold_distance) + + self.assertIsInstance(filtered_df, pd.DataFrame) + self.assertLessEqual(len(filtered_df), len(df)) + + def test_spatial_sampling_ordering(self): + data = { + "geometry": [ + "POINT (-74.006 40.7128)", + "POINT (-75.006 41.7128)", + "POINT (-76.006 42.7128)", + "POINT (-77.006 43.7128)", + ], + "captured_at": [1, 2, 3, 4], + "sequence_id": ["1", "1", "1", "1"], + } + df = pd.DataFrame(data) + df["geometry"] = df["geometry"].apply(wkt.loads) + + interval_length = 0.1 + filtered_gdf = spatial_sampling(df, interval_length) + + self.assertTrue(filtered_gdf["captured_at"].is_monotonic_decreasing) + + def test_spatial_sampling_with_sequence(self): + threshold_distance = 0.01 + filtered_df = spatial_sampling(self.fixture_df, threshold_distance) + self.assertIsInstance(filtered_df, pd.DataFrame) + self.assertLess(len(filtered_df), len(self.fixture_df)) + + filtered_df.reset_index(drop=True, inplace=True) + for i in range(len(filtered_df) - 1): + geom1 = filtered_df.loc[i, "geometry"] + geom2 = filtered_df.loc[i + 1, "geometry"] + + distance = geom1.distance(geom2) + + self.assertLess(distance, threshold_distance) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_tutorial_arbitrary_geometry_footprint.py b/mapswipe_workers/tests/unittests/test_tutorial_arbitrary_geometry_footprint.py new file mode 100644 index 000000000..3d20b6289 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_tutorial_arbitrary_geometry_footprint.py @@ -0,0 +1,27 @@ +import os +import unittest + +from mapswipe_workers.project_types import FootprintTutorial +from tests.fixtures import FIXTURE_DIR, get_fixture + + +class TestTutorial(unittest.TestCase): + def test_init_arbitrary_geometry_footprint_project(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "footprint.json") + ) + self.assertIsNotNone(FootprintTutorial(tutorial_draft=tutorial_draft)) + + def test_create_arbitrary_geometry_footprint_tasks(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "footprint.json") + ) + tutorial = FootprintTutorial(tutorial_draft=tutorial_draft) + tutorial.create_tutorial_groups() + tutorial.create_tutorial_tasks() + self.assertTrue(tutorial.groups) + self.assertTrue(tutorial.tasks) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_tutorial_street.py b/mapswipe_workers/tests/unittests/test_tutorial_street.py new file mode 100644 index 000000000..6dd9b0127 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_tutorial_street.py @@ -0,0 +1,27 @@ +import os +import unittest + +from mapswipe_workers.project_types import StreetTutorial +from tests.fixtures import FIXTURE_DIR, get_fixture + + +class TestTutorial(unittest.TestCase): + def test_init_street_tutorial(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "street.json") + ) + self.assertIsNotNone(StreetTutorial(tutorial_draft=tutorial_draft)) + + def test_create_street_tasks(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "street.json") + ) + tutorial = StreetTutorial(tutorial_draft=tutorial_draft) + tutorial.create_tutorial_groups() + tutorial.create_tutorial_tasks() + self.assertTrue(tutorial.groups) + self.assertTrue(tutorial.tasks) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_tutorial_tile_change_detection.py b/mapswipe_workers/tests/unittests/test_tutorial_tile_change_detection.py new file mode 100644 index 000000000..e394fd607 --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_tutorial_tile_change_detection.py @@ -0,0 +1,27 @@ +import os +import unittest + +from mapswipe_workers.project_types import ChangeDetectionTutorial +from tests.fixtures import FIXTURE_DIR, get_fixture + + +class TestTutorial(unittest.TestCase): + def test_init_tile_change_detection_project(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "change_detection.json") + ) + self.assertIsNotNone(ChangeDetectionTutorial(tutorial_draft=tutorial_draft)) + + def test_create_tile_change_detection_tasks(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "change_detection.json") + ) + tutorial = ChangeDetectionTutorial(tutorial_draft=tutorial_draft) + tutorial.create_tutorial_groups() + tutorial.create_tutorial_tasks() + self.assertTrue(tutorial.groups) + self.assertTrue(tutorial.tasks) + + +if __name__ == "__main__": + unittest.main() diff --git a/mapswipe_workers/tests/unittests/test_tutorial.py b/mapswipe_workers/tests/unittests/test_tutorial_tile_classification.py similarity index 100% rename from mapswipe_workers/tests/unittests/test_tutorial.py rename to mapswipe_workers/tests/unittests/test_tutorial_tile_classification.py diff --git a/mapswipe_workers/tests/unittests/test_tutorial_tile_completeness.py b/mapswipe_workers/tests/unittests/test_tutorial_tile_completeness.py new file mode 100644 index 000000000..972c412ca --- /dev/null +++ b/mapswipe_workers/tests/unittests/test_tutorial_tile_completeness.py @@ -0,0 +1,27 @@ +import os +import unittest + +from mapswipe_workers.project_types import CompletenessTutorial +from tests.fixtures import FIXTURE_DIR, get_fixture + + +class TestTutorial(unittest.TestCase): + def test_init_tile_completeness_project(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "completeness.json") + ) + self.assertIsNotNone(CompletenessTutorial(tutorial_draft=tutorial_draft)) + + def test_create_tile_completeness_tasks(self): + tutorial_draft = get_fixture( + os.path.join(FIXTURE_DIR, "tutorialDrafts", "completeness.json") + ) + tutorial = CompletenessTutorial(tutorial_draft=tutorial_draft) + tutorial.create_tutorial_groups() + tutorial.create_tutorial_tasks() + self.assertTrue(tutorial.groups) + self.assertTrue(tutorial.tasks) + + +if __name__ == "__main__": + unittest.main() diff --git a/postgres/initdb.sql b/postgres/initdb.sql index ce9b97197..f954d3a8c 100644 --- a/postgres/initdb.sql +++ b/postgres/initdb.sql @@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS tasks ( project_id varchar, group_id varchar, task_id varchar, - geom geometry(MULTIPOLYGON, 4326), + geom geometry(Geometry, 4326), project_type_specifics json, PRIMARY KEY (project_id, group_id, task_id), FOREIGN KEY (project_id) REFERENCES projects (project_id), diff --git a/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql b/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql new file mode 100644 index 000000000..97ecc298f --- /dev/null +++ b/postgres/scripts/v2_to_v3/08_change_geom_type_for_tasks.sql @@ -0,0 +1,12 @@ +/* + * This script updates the `tasks` table by adjusting the following column: + * - `geom`: Stores now not only polygon geometries but all geometry types (e.g. Point, LineString). + * + * Existing entries for `geom` are not affected by this change. + * + * The new street project type requires Point geometries to store the image location. + * + */ + + +ALTER TABLE tasks ALTER COLUMN geom SET DATA TYPE geometry(Geometry, 4326); \ No newline at end of file