Skip to content

Commit 1b66bda

Browse files
yannchevalieryannchevalier
committed
Pull request #77: [Offscreen] Display solver stroke in recognition feedback.
Merge in SDK/iink_sdk-additional-examples-android from display-solver-stroke to master * commit '111550ccc9287a2c99e9c23eca0fefba3035de36': Bump to 4.1.3 [Offscreen] Display solver stroke in recognition feedback.
2 parents d03760e + 111550c commit 1b66bda

File tree

4 files changed

+116
-6
lines changed

4 files changed

+116
-6
lines changed

‎samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/ink/serialization/jiix/Solver.kt‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
// Copyright @ MyScript. All rights reserved.
2+
3+
package com.myscript.iink.demo.ink.serialization.jiix
4+
5+
import com.google.gson.annotations.SerializedName
6+
7+
data class ItemSolver(
8+
@SerializedName("timestamp")
9+
val timestamp: String? = null,
10+
@SerializedName("X")
11+
val X: List<Float>? = null,
12+
@SerializedName("Y")
13+
val Y: List<Float>? = null,
14+
)
15+
16+
data class OperandSolver(
17+
@SerializedName("label")
18+
val label: String? = null,
19+
@SerializedName("bounding-box")
20+
val boundingBox: BoundingBox? = null,
21+
@SerializedName("solver-output")
22+
val solverOutput: Boolean = false,
23+
@SerializedName("items")
24+
val items: List<ItemSolver>? = null
25+
)
26+
27+
data class ExpressionSolver(
28+
@SerializedName("operands")
29+
val operands: List<OperandSolver>? = null
30+
)
31+
32+
data class SolverRoot(
33+
@SerializedName("type")
34+
val type: String,
35+
@SerializedName("expressions")
36+
val expressions: List<ExpressionSolver>? = null
37+
)

‎samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/ui/InkViewModel.kt‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
package com.myscript.iink.demo.inksample.ui
44

5+
import android.graphics.PointF
56
import android.util.DisplayMetrics
67
import android.util.Log
78
import androidx.annotation.VisibleForTesting
@@ -25,9 +26,8 @@ import com.myscript.iink.MimeType
2526
import com.myscript.iink.OffscreenEditor
2627
import com.myscript.iink.OffscreenGestureAction
2728
import com.myscript.iink.demo.ink.serialization.jiix.BoundingBox
28-
import com.myscript.iink.demo.ink.serialization.jiix.Element
2929
import com.myscript.iink.demo.ink.serialization.jiix.RecognitionRoot
30-
import com.myscript.iink.demo.ink.serialization.jiix.Word
30+
import com.myscript.iink.demo.ink.serialization.jiix.SolverRoot
3131
import com.myscript.iink.demo.ink.serialization.jiix.convertPointerEvent
3232
import com.myscript.iink.demo.ink.serialization.jiix.toBrush
3333
import com.myscript.iink.demo.ink.serialization.jiix.toPointerEvents
@@ -49,13 +49,19 @@ import java.util.concurrent.Executors
4949

5050
enum class BlockType {
5151
TEXT,
52-
MATH
52+
MATH,
53+
SOLVING
5354
}
5455

56+
data class Stroke(
57+
val points: List<PointF>
58+
)
59+
5560
data class RecognitionItem(
5661
val text: String,
5762
val type: BlockType,
58-
val boundingBox: BoundingBox?
63+
val boundingBox: BoundingBox?,
64+
val strokes: List<Stroke> = emptyList()
5965
)
6066

6167
data class RecognitionFeedback(
@@ -341,6 +347,47 @@ class InkViewModel(
341347

342348
recognitionRoot.elements?.forEach { element ->
343349
if (element.expressions != null) {
350+
351+
// If possible, get solver output and strokes.
352+
if (offscreenEditor?.mathSolverController?.getAvailableActions(element.id)?.contains("numerical-computation") == true) {
353+
val solverJIIX = offscreenEditor?.mathSolverController?.getActionOutput(element.id, "numerical-computation", MimeType.JIIX, engine?.createParameterSet()?.apply {
354+
setBoolean("export.jiix.strokes", true);
355+
})
356+
357+
val solverRoot = Gson().fromJson(solverJIIX, SolverRoot::class.java)
358+
if (solverRoot.type == "Math") {
359+
solverRoot.expressions?.forEach { expression ->
360+
expression.operands?.forEach { operand ->
361+
if (operand.solverOutput) {
362+
val strokes = mutableListOf<Stroke>()
363+
operand.items?.forEach loop@{ item ->
364+
val xPoints = item.X ?: return@loop
365+
val yPoints = item.Y ?: return@loop
366+
val converter = converter ?: return@loop
367+
368+
val points = mutableListOf<PointF>()
369+
for (index in xPoints.indices) {
370+
points.add(PointF(
371+
converter.x_mm2px(xPoints[index]),
372+
converter.y_mm2px(yPoints[index])))
373+
}
374+
strokes.add(Stroke(points))
375+
}
376+
377+
recognitionResult.add(
378+
RecognitionItem(
379+
text = operand.label ?: "",
380+
type = BlockType.SOLVING,
381+
boundingBox = operand.boundingBox?.toScreenCoordinates(converter),
382+
strokes = strokes
383+
)
384+
)
385+
}
386+
}
387+
}
388+
}
389+
}
390+
344391
recognitionResult.add(RecognitionItem(
345392
text = "",
346393
type = BlockType.MATH,

‎samples/offscreen-interactivity/src/main/java/com/myscript/iink/demo/inksample/ui/RecognitionItemView.kt‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import android.content.Context
66
import android.graphics.Canvas
77
import android.graphics.Color
88
import android.graphics.Paint
9+
import android.graphics.Path
910
import android.graphics.Typeface
1011
import android.text.TextPaint
1112
import android.view.View
@@ -29,6 +30,14 @@ class RecognitionItemView(context: Context, private val item: RecognitionItem) :
2930
typeface = Typeface.create(Typeface.DEFAULT, Typeface.ITALIC)
3031
}
3132

33+
private val solvingPaint = Paint(Paint.ANTI_ALIAS_FLAG).apply {
34+
color = Color.GRAY
35+
style = Paint.Style.STROKE
36+
strokeWidth = 4f
37+
}
38+
39+
private val solvingPath = Path()
40+
3241
override fun onDraw(canvas: Canvas) {
3342
super.onDraw(canvas)
3443

@@ -42,6 +51,7 @@ class RecognitionItemView(context: Context, private val item: RecognitionItem) :
4251
color = when(item.type) {
4352
BlockType.TEXT -> TEXT_COLOR
4453
BlockType.MATH -> MATH_COLOR
54+
BlockType.SOLVING -> SOLVER_COLOR
4555
}
4656
}
4757
)
@@ -59,21 +69,37 @@ class RecognitionItemView(context: Context, private val item: RecognitionItem) :
5969
when(item.type) {
6070
BlockType.TEXT -> "abc"
6171
BlockType.MATH -> "Σ"
72+
BlockType.SOLVING -> "="
6273
},
6374
box.x + 5,
6475
box.y + (textPaint.descent() - textPaint.ascent()) / 2,
6576
typePaint.apply {
6677
color = when(item.type) {
6778
BlockType.TEXT -> TEXT_COLOR
6879
BlockType.MATH -> MATH_COLOR
80+
BlockType.SOLVING -> SOLVER_COLOR
6981
}
7082
}
7183
)
84+
85+
if (item.type == BlockType.SOLVING && item.strokes.isNotEmpty()) {
86+
item.strokes.forEach { stroke ->
87+
if (stroke.points.isNotEmpty()) {
88+
solvingPath.reset()
89+
solvingPath.moveTo(stroke.points[0].x, stroke.points[0].y)
90+
for (i in 1 until stroke.points.size) {
91+
solvingPath.lineTo(stroke.points[i].x, stroke.points[i].y)
92+
}
93+
canvas.drawPath(solvingPath, solvingPaint)
94+
}
95+
}
96+
}
7297
}
7398
}
7499

75100
companion object {
76101
private const val TEXT_COLOR = Color.RED
77102
private const val MATH_COLOR = Color.BLUE
103+
private const val SOLVER_COLOR = Color.GREEN
78104
}
79105
}

‎samples/settings.gradle‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55

66
rootProject.name = 'myscript.iink.samples.android.java'
77

8-
gradle.ext.iinkVersionCode = 4100
9-
gradle.ext.iinkVersionName = "4.1.0"
8+
gradle.ext.iinkVersionCode = 4130
9+
gradle.ext.iinkVersionName = "4.1.3"
1010
gradle.ext.iinkResourcesURL = "https://download.myscript.com/iink/recognitionAssets_iink_4.1"
1111

1212
def projects = [

0 commit comments

Comments
 (0)