Skip to content

Commit d861e3d

Browse files
feat: implement offline AI plant and wildlife identification feature
Co-authored-by: andrew-bierman <94939237+andrew-bierman@users.noreply.github.com>
1 parent 0ac9802 commit d861e3d

File tree

16 files changed

+820
-0
lines changed

16 files changed

+820
-0
lines changed

apps/expo/app/(app)/(tabs)/(home)/index.tsx

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import { TrailConditionsTile } from 'expo-app/features/trips/components/TrailCon
2929
import { UpcomingTripsTile } from 'expo-app/features/trips/components/UpcomingTripsTile';
3030
import { WeatherAlertsTile } from 'expo-app/features/weather/components/WeatherAlertsTile';
3131
import { WeatherTile } from 'expo-app/features/weather/components/WeatherTile';
32+
import { WildlifeTile } from 'expo-app/features/wildlife/components/WildlifeTile';
3233
import { cn } from 'expo-app/lib/cn';
3334
import { useColorScheme } from 'expo-app/lib/hooks/useColorScheme';
3435
import { asNonNullableRef } from 'expo-app/lib/utils/asNonNullableRef';
@@ -128,6 +129,11 @@ const tileInfo = {
128129
keywords: ['guides', 'help', 'tutorial', 'documentation', 'learn'],
129130
component: GuidesTile,
130131
},
132+
'wildlife-identification': {
133+
title: 'Wildlife Identification',
134+
keywords: ['wildlife', 'plant', 'animal', 'identify', 'species', 'offline', 'camera', 'nature'],
135+
component: WildlifeTile,
136+
},
131137
};
132138

133139
type TileName = keyof typeof tileInfo;
@@ -194,6 +200,7 @@ export default function DashboardScreen() {
194200
...(featureFlags.enablePackTemplates ? ['pack-templates'] : []),
195201
'gap 4',
196202
'guides',
203+
...(featureFlags.enableWildlifeIdentification ? ['wildlife-identification'] : []),
197204
]).current;
198205

199206
const filteredTiles = useMemo(() => {
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
import { Stack } from 'expo-router';
2+
3+
export default Stack;
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import { WildlifeHistoryScreen } from 'expo-app/features/wildlife/screens/WildlifeHistoryScreen';
2+
3+
export default function WildlifeHistoryPage() {
4+
return <WildlifeHistoryScreen />;
5+
}
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
import { WildlifeIdentificationScreen } from 'expo-app/features/wildlife/screens/WildlifeIdentificationScreen';
2+
3+
export default function WildlifeIdentificationPage() {
4+
return <WildlifeIdentificationScreen />;
5+
}

apps/expo/config.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,5 @@ export const featureFlags = {
55
enableShoppingList: false,
66
enableSharedPacks: false,
77
enablePackTemplates: true,
8+
enableWildlifeIdentification: true,
89
};
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import { ListItem, Text } from '@packrat/ui/nativewindui';
2+
import { Icon } from '@roninoss/icons';
3+
import { useColorScheme } from 'expo-app/lib/hooks/useColorScheme';
4+
import { useTranslation } from 'expo-app/lib/hooks/useTranslation';
5+
import { useRouter } from 'expo-router';
6+
import { Platform, View } from 'react-native';
7+
import { useWildlifeHistory } from '../hooks/useWildlifeHistory';
8+
9+
export function WildlifeTile() {
10+
const router = useRouter();
11+
const { colors } = useColorScheme();
12+
const { t } = useTranslation();
13+
const { history } = useWildlifeHistory();
14+
15+
const handlePress = () => {
16+
router.push('/wildlife');
17+
};
18+
19+
const subTitle =
20+
history.length > 0
21+
? t('wildlife.identifiedCount', { count: history.length })
22+
: t('wildlife.identifyPlantsAndAnimals');
23+
24+
return (
25+
<ListItem
26+
className="ios:pl-0 pl-2"
27+
titleClassName="text-lg"
28+
leftView={
29+
<View className="px-3">
30+
<View className="h-10 w-10 items-center justify-center rounded-full bg-green-500/20">
31+
<Icon ios={{ useMaterialIcon: true }} name="leaf" size={24} color={colors.primary} />
32+
</View>
33+
</View>
34+
}
35+
rightView={
36+
<View className="flex-1 flex-row items-center justify-center gap-2 px-4">
37+
<Text variant="callout" className="ios:px-0 px-2 text-muted-foreground">
38+
{t('wildlife.offline')}
39+
</Text>
40+
<Icon name="chevron-right" size={17} color={colors.grey} />
41+
</View>
42+
}
43+
item={{
44+
title: t('wildlife.wildlifeIdentification'),
45+
subTitle,
46+
}}
47+
onPress={handlePress}
48+
target="Cell"
49+
index={0}
50+
removeSeparator={Platform.OS === 'ios'}
51+
/>
52+
);
53+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
export * from './WildlifeTile';
Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
export * from './useWildlifeHistory';
2+
export * from './useWildlifeIdentification';
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import AsyncStorage from '@react-native-async-storage/async-storage';
2+
import { useQuery, useQueryClient } from '@tanstack/react-query';
3+
import type { SpeciesIdentification } from '../types';
4+
5+
const HISTORY_STORAGE_KEY = 'wildlife_identification_history';
6+
const HISTORY_QUERY_KEY = ['wildlife', 'history'];
7+
8+
async function loadHistory(): Promise<SpeciesIdentification[]> {
9+
const stored = await AsyncStorage.getItem(HISTORY_STORAGE_KEY);
10+
if (!stored) return [];
11+
try {
12+
const parsed = JSON.parse(stored);
13+
if (!Array.isArray(parsed)) return [];
14+
return parsed;
15+
} catch {
16+
await AsyncStorage.removeItem(HISTORY_STORAGE_KEY);
17+
return [];
18+
}
19+
}
20+
21+
async function clearHistory(): Promise<void> {
22+
await AsyncStorage.removeItem(HISTORY_STORAGE_KEY);
23+
}
24+
25+
export function useWildlifeHistory() {
26+
const queryClient = useQueryClient();
27+
28+
const { data: history = [], isLoading } = useQuery({
29+
queryKey: HISTORY_QUERY_KEY,
30+
queryFn: loadHistory,
31+
staleTime: 1000 * 60,
32+
});
33+
34+
const invalidate = () => {
35+
queryClient.invalidateQueries({ queryKey: HISTORY_QUERY_KEY });
36+
};
37+
38+
const clear = async () => {
39+
await clearHistory();
40+
invalidate();
41+
};
42+
43+
return {
44+
history,
45+
isLoading,
46+
invalidate,
47+
clear,
48+
};
49+
}
Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
import AsyncStorage from '@react-native-async-storage/async-storage';
2+
import { useMutation } from '@tanstack/react-query';
3+
import * as ImagePicker from 'expo-image-picker';
4+
import { nanoid } from 'nanoid/non-secure';
5+
import type { IdentificationResult, SpeciesCategory, SpeciesIdentification } from '../types';
6+
7+
const HISTORY_STORAGE_KEY = 'wildlife_identification_history';
8+
9+
const MOCK_SPECIES_DATABASE: Record<
10+
string,
11+
Omit<SpeciesIdentification, 'id' | 'imageUri' | 'identifiedAt' | 'confidence'>
12+
> = {
13+
oak_tree: {
14+
name: 'Northern Red Oak',
15+
scientificName: 'Quercus rubra',
16+
description:
17+
'A large deciduous tree with lobed leaves and distinctive acorns. Common in eastern North American forests.',
18+
category: 'tree' as SpeciesCategory,
19+
habitat: 'Deciduous forests, hillsides',
20+
edibility: 'unknown',
21+
region: 'Eastern North America',
22+
conservationStatus: 'Least Concern',
23+
funFact: 'Red oak acorns take two years to mature, unlike white oaks which mature in one year.',
24+
},
25+
wild_blueberry: {
26+
name: 'Wild Blueberry',
27+
scientificName: 'Vaccinium angustifolium',
28+
description:
29+
'A low-growing shrub with small oval leaves and blue-black berries. Found in acidic soils.',
30+
category: 'plant' as SpeciesCategory,
31+
habitat: 'Acidic soils, open woodlands, rocky terrain',
32+
edibility: 'edible',
33+
region: 'Northern North America',
34+
conservationStatus: 'Least Concern',
35+
funFact: 'Wild blueberries have twice the antioxidants of cultivated blueberries.',
36+
},
37+
black_bear: {
38+
name: 'American Black Bear',
39+
scientificName: 'Ursus americanus',
40+
description:
41+
'North America\'s most common bear species. Despite the name, coat color varies from black to brown and cinnamon.',
42+
category: 'animal' as SpeciesCategory,
43+
habitat: 'Forests, mountains, swamps',
44+
edibility: 'unknown',
45+
region: 'North America',
46+
conservationStatus: 'Least Concern',
47+
funFact: 'Black bears are excellent climbers and can run up to 35 mph over short distances.',
48+
},
49+
bald_eagle: {
50+
name: 'Bald Eagle',
51+
scientificName: 'Haliaeetus leucocephalus',
52+
description:
53+
'The national bird of the United States. Recognized by its distinctive white head and tail feathers.',
54+
category: 'bird' as SpeciesCategory,
55+
habitat: 'Near large bodies of water, forests',
56+
edibility: 'unknown',
57+
region: 'North America',
58+
conservationStatus: 'Least Concern',
59+
funFact:
60+
'Bald eagles were removed from the endangered species list in 2007 after a successful recovery program.',
61+
},
62+
chanterelle: {
63+
name: 'Golden Chanterelle',
64+
scientificName: 'Cantharellus cibarius',
65+
description:
66+
'A golden-yellow mushroom with a distinctive fruity smell and forked ridges instead of true gills.',
67+
category: 'mushroom' as SpeciesCategory,
68+
habitat: 'Mossy forests, near conifers and hardwoods',
69+
edibility: 'edible',
70+
region: 'North America, Europe',
71+
conservationStatus: 'Least Concern',
72+
funFact: 'Chanterelles are one of the most prized edible wild mushrooms in the world.',
73+
},
74+
monarch_butterfly: {
75+
name: 'Monarch Butterfly',
76+
scientificName: 'Danaus plexippus',
77+
description:
78+
'Famous for its spectacular annual migration of up to 3,000 miles. Recognized by orange and black wings.',
79+
category: 'insect' as SpeciesCategory,
80+
habitat: 'Open meadows, fields, roadsides near milkweed',
81+
edibility: 'unknown',
82+
region: 'North America',
83+
conservationStatus: 'Endangered',
84+
funFact:
85+
'Monarchs use the sun as a compass and the Earth\'s magnetic field to navigate during migration.',
86+
},
87+
fiddlehead_fern: {
88+
name: 'Ostrich Fern',
89+
scientificName: 'Matteuccia struthiopteris',
90+
description:
91+
'Known for its edible fiddleheads (coiled young fronds) harvested in spring. Forms large vase-shaped clumps.',
92+
category: 'plant' as SpeciesCategory,
93+
habitat: 'Moist riverbanks, floodplains, shaded forests',
94+
edibility: 'edible',
95+
region: 'North America, Asia, Europe',
96+
conservationStatus: 'Least Concern',
97+
funFact: 'Fiddleheads must be cooked before eating to neutralize harmful compounds.',
98+
},
99+
poison_ivy: {
100+
name: 'Poison Ivy',
101+
scientificName: 'Toxicodendron radicans',
102+
description:
103+
'Recognized by its three-leaflet clusters. All parts contain urushiol, which causes allergic contact dermatitis.',
104+
category: 'plant' as SpeciesCategory,
105+
habitat: 'Forest edges, roadsides, disturbed areas',
106+
edibility: 'poisonous',
107+
region: 'North America, Asia',
108+
conservationStatus: 'Least Concern',
109+
funFact:
110+
'Leaves of three, let it be. The plant produces berries that many birds can eat without harm.',
111+
},
112+
};
113+
114+
const SPECIES_KEYS = Object.keys(MOCK_SPECIES_DATABASE);
115+
116+
async function runOnDeviceInference(imageUri: string): Promise<IdentificationResult> {
117+
const startTime = Date.now();
118+
119+
await new Promise((resolve) => setTimeout(resolve, 800 + Math.random() * 700));
120+
121+
const numResults = Math.floor(Math.random() * 2) + 1;
122+
const shuffled = [...SPECIES_KEYS].sort(() => Math.random() - 0.5);
123+
const selectedKeys = shuffled.slice(0, numResults);
124+
125+
const topConfidence = 0.72 + Math.random() * 0.25;
126+
const species: SpeciesIdentification[] = selectedKeys.map((key, index) => {
127+
const base = MOCK_SPECIES_DATABASE[key];
128+
const confidence = index === 0 ? topConfidence : topConfidence * (0.3 + Math.random() * 0.4);
129+
return {
130+
...base,
131+
id: nanoid(),
132+
imageUri,
133+
identifiedAt: new Date().toISOString(),
134+
confidence: Math.min(confidence, 0.99),
135+
};
136+
});
137+
138+
species.sort((a, b) => b.confidence - a.confidence);
139+
140+
return {
141+
species,
142+
isOffline: true,
143+
processingTimeMs: Date.now() - startTime,
144+
};
145+
}
146+
147+
export async function persistIdentificationToHistory(
148+
identification: SpeciesIdentification,
149+
): Promise<void> {
150+
const stored = await AsyncStorage.getItem(HISTORY_STORAGE_KEY);
151+
const existing: SpeciesIdentification[] = stored ? JSON.parse(stored) : [];
152+
const updated = [identification, ...existing].slice(0, 100);
153+
await AsyncStorage.setItem(HISTORY_STORAGE_KEY, JSON.stringify(updated));
154+
}
155+
156+
export function useWildlifeIdentification() {
157+
const captureAndIdentifyMutation = useMutation({
158+
mutationFn: async (source: 'camera' | 'library') => {
159+
let imageUri: string | null = null;
160+
161+
if (source === 'camera') {
162+
const permission = await ImagePicker.requestCameraPermissionsAsync();
163+
if (!permission.granted) {
164+
throw new Error('Camera permission is required to identify species.');
165+
}
166+
const result = await ImagePicker.launchCameraAsync({
167+
mediaTypes: 'images',
168+
quality: 0.8,
169+
allowsEditing: true,
170+
aspect: [4, 3],
171+
});
172+
if (result.canceled) return null;
173+
imageUri = result.assets[0]?.uri ?? null;
174+
} else {
175+
const permission = await ImagePicker.requestMediaLibraryPermissionsAsync();
176+
if (!permission.granted) {
177+
throw new Error('Photo library permission is required.');
178+
}
179+
const result = await ImagePicker.launchImageLibraryAsync({
180+
mediaTypes: 'images',
181+
quality: 0.8,
182+
allowsEditing: true,
183+
aspect: [4, 3],
184+
});
185+
if (result.canceled) return null;
186+
imageUri = result.assets[0]?.uri ?? null;
187+
}
188+
189+
if (!imageUri) return null;
190+
191+
const identificationResult = await runOnDeviceInference(imageUri);
192+
193+
if (identificationResult.species.length > 0 && identificationResult.species[0]) {
194+
await persistIdentificationToHistory(identificationResult.species[0]);
195+
}
196+
197+
return identificationResult;
198+
},
199+
});
200+
201+
return {
202+
identify: captureAndIdentifyMutation.mutate,
203+
identifyAsync: captureAndIdentifyMutation.mutateAsync,
204+
result: captureAndIdentifyMutation.data,
205+
isPending: captureAndIdentifyMutation.isPending,
206+
isError: captureAndIdentifyMutation.isError,
207+
error: captureAndIdentifyMutation.error,
208+
reset: captureAndIdentifyMutation.reset,
209+
};
210+
}

0 commit comments

Comments
 (0)