oist · kngwyu · Mar 19, 2025 · Mar 18, 2025
diff --git a/apps-config.json b/apps-config.json
@@ -6,6 +6,7 @@
       "basicServer",
       "basicCharger",
       "basicQRTransmitter",
-      "basicQRReceiver"
+      "basicQRReceiver",
+      "handsOnApp"
   ]
 }
diff --git a/apps/basicSubscriber/src/main/java/jp/oist/abcvlib/basicsubscriber/MainActivity.java b/apps/basicSubscriber/src/main/java/jp/oist/abcvlib/basicsubscriber/MainActivity.java
@@ -4,7 +4,6 @@
 import android.graphics.Bitmap;
 import android.media.AudioTimestamp;
 import android.os.Bundle;
-import android.util.Log;
 
 import org.tensorflow.lite.support.image.TensorImage;
 import org.tensorflow.lite.support.label.Category;
@@ -17,8 +16,6 @@
 import java.util.concurrent.TimeUnit;
 
 import jp.oist.abcvlib.core.AbcvlibActivity;
-import jp.oist.abcvlib.core.AbcvlibLooper;
-import jp.oist.abcvlib.core.IOReadyListener;
 import jp.oist.abcvlib.core.inputs.PublisherManager;
 import jp.oist.abcvlib.core.inputs.microcontroller.BatteryData;
 import jp.oist.abcvlib.core.inputs.microcontroller.BatteryDataSubscriber;

diff --git a/apps/handsOnApp/build.gradle b/apps/handsOnApp/build.gradle
@@ -0,0 +1,3 @@
+android {
+    namespace 'jp.oist.abcvlib.handsOnApp'
+}
diff --git a/apps/handsOnApp/src/main/AndroidManifest.xml b/apps/handsOnApp/src/main/AndroidManifest.xml
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    package="jp.oist.abcvlib.handsOnApp">
+
+    <application
+        android:theme="@style/Theme.AppCompat"
+        android:extractNativeLibs="true"
+        android:label="@string/app_name">
+        <uses-library
+            android:name="com.android.future.usb.accessory"
+            android:required="false" />
+        <activity
+            android:name="jp.oist.abcvlib.handsOnApp.MainActivity"
+            android:exported="true"
+            android:launchMode="singleTask">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+            <intent-filter>
+                <action android:name="android.hardware.usb.action.USB_ACCESSORY_ATTACHED" />
+            </intent-filter>
+            <meta-data
+                android:name="android.hardware.usb.action.USB_ACCESSORY_ATTACHED"
+                android:resource="@xml/accessory_filter" />
+            <intent-filter>
+                <action android:name="android.hardware.usb.action.USB_DEVICE_ATTACHED" />
+            </intent-filter>
+            <meta-data
+                android:name="android.hardware.usb.action.USB_DEVICE_ATTACHED"
+                android:resource="@xml/device_filter" />
+
+        </activity>
+    </application>
+
+</manifest>
diff --git a/apps/handsOnApp/src/main/java/jp/oist/abcvlib/handsOnApp/DebugInfoViewer.java b/apps/handsOnApp/src/main/java/jp/oist/abcvlib/handsOnApp/DebugInfoViewer.java
@@ -0,0 +1,46 @@
+package jp.oist.abcvlib.handsOnApp;
+
+import android.app.Activity;
+import android.graphics.Bitmap;
+import android.widget.ImageView;
+import android.widget.TextView;
+
+public class DebugInfoViewer implements Runnable {
+    private final Activity activity;
+    private final TextView textView1;
+    private final TextView textView2;
+    private final TextView textView3;
+    private final ImageView imageView;
+    volatile String text1;
+    volatile String text2;
+    volatile String text3;
+    volatile Bitmap image;
+
+
+    public DebugInfoViewer(
+            Activity activityInit,
+            TextView textViewInit1,
+            TextView textViewInit2,
+            TextView textViewInit3,
+            ImageView imageViewInit
+    ){
+        activity = activityInit;
+        textView1 = textViewInit1;
+        textView2 = textViewInit2;
+        textView3 = textViewInit3;
+        imageView = imageViewInit;
+        image = null;
+    }
+
+    @Override
+    public void run() {
+        activity.runOnUiThread(() -> {
+            this.textView1.setText(text1);
+            this.textView2.setText(text2);
+            this.textView3.setText(text3);
+            if (image != null) {
+                imageView.setImageBitmap(image);
+            }
+        });
+    }
+}
diff --git a/apps/handsOnApp/src/main/java/jp/oist/abcvlib/handsOnApp/MainActivity.java b/apps/handsOnApp/src/main/java/jp/oist/abcvlib/handsOnApp/MainActivity.java
@@ -0,0 +1,140 @@
+package jp.oist.abcvlib.handsOnApp;
+
+import android.graphics.Bitmap;
+import android.graphics.Matrix;
+import android.os.Bundle;
+import android.widget.ImageView;
+import android.widget.TextView;
+import android.os.Handler;
+import android.os.Looper;
+
+import org.tensorflow.lite.support.image.TensorImage;
+import org.tensorflow.lite.support.label.Category;
+import org.tensorflow.lite.task.vision.detector.Detection;
+
+import java.util.List;
+
+import jp.oist.abcvlib.core.AbcvlibActivity;
+import jp.oist.abcvlib.core.inputs.microcontroller.BatteryData;
+import jp.oist.abcvlib.core.inputs.microcontroller.BatteryDataSubscriber;
+import jp.oist.abcvlib.core.inputs.PublisherManager;
+import jp.oist.abcvlib.core.inputs.phone.ObjectDetectorData;
+import jp.oist.abcvlib.core.inputs.phone.ObjectDetectorDataSubscriber;
+import jp.oist.abcvlib.core.inputs.microcontroller.WheelData;
+import jp.oist.abcvlib.core.inputs.microcontroller.WheelDataSubscriber;
+import jp.oist.abcvlib.util.SerialCommManager;
+import jp.oist.abcvlib.util.SerialReadyListener;
+import jp.oist.abcvlib.util.UsbSerial;
+
+/**
+ * Demo app for Handson
+ *
+ * @author Yuji Kanagawa https://github.com/kngwyu
+ */
+public class MainActivity extends AbcvlibActivity implements BatteryDataSubscriber, SerialReadyListener, WheelDataSubscriber, ObjectDetectorDataSubscriber {
+    private final Handler handler = new Handler(Looper.getMainLooper());
+    private DebugInfoViewer debugInfo;
+    private PublisherManager publisherManager;
+    private int countL = 0;
+    private int countR = 0;
+    private int nLoopCalled = 0;
+    private String imageLabel = "Nothing";
+
+
+    @Override
+    protected void abcvlibMainLoop() {
+        nLoopCalled += 1;
+        // Example code for controlling robots
+        // Set wheel output
+        // Stop when detected something
+        if (!imageLabel.equals("Nothing")) {
+            outputs.setWheelOutput(0.0f, 0.0f, false, false);
+        } else if (100 < nLoopCalled && nLoopCalled < 120) {
+            outputs.setWheelOutput(0.0f, 1.0f, false, false);
+        } else if (120 < nLoopCalled && nLoopCalled < 140) {
+            outputs.setWheelOutput(-1.0f, -1.0f, false, false);
+        } else {
+            outputs.setWheelOutput(1.0f, 1.0f, false, false);
+        }
+        debugInfo.text1 = String.format("MainLoopCount: %d", nLoopCalled);
+    }
+
+    @Override
+    protected void onCreate(Bundle savedInstanceState) {
+        // Setup Android GUI. Point this method to your main activity xml file or corresponding int
+        // ID within the R class
+        setContentView(R.layout.activity_main);
+
+        final TextView textView1 = (TextView) findViewById(R.id.description);
+        final TextView textView2 = (TextView) findViewById(R.id.description2);
+        final TextView textView3 = (TextView) findViewById(R.id.description3);
+        final ImageView imageView = (ImageView) findViewById(R.id.demoImage);
+        // Set up debugInfo
+        debugInfo = new DebugInfoViewer(this, textView1, textView2, textView3, imageView);
+        Runnable updateTask = new Runnable() {
+            @Override
+            public void run() {
+                handler.post(debugInfo);
+                handler.postDelayed(this, 100); // Schedule the next update
+            }
+        };
+        handler.post(updateTask);
+        super.onCreate(savedInstanceState);
+    }
+
+    @Override
+    public void onSerialReady(UsbSerial usbSerial) {
+        publisherManager = new PublisherManager();
+
+        WheelData wheelData = new WheelData.Builder(this, publisherManager)
+                .setBufferLength(50)
+                .setExpWeight(0.01).build();
+        wheelData.addSubscriber(this);
+        BatteryData batteryData = new BatteryData.Builder(this, publisherManager).build();
+        batteryData.addSubscriber(this);
+        ObjectDetectorData detectorData = new ObjectDetectorData.Builder(this, publisherManager, this)
+                .setModel("efficientdet-lite1.tflite")
+                .build();
+        detectorData.addSubscriber(this);
+        setSerialCommManager(new SerialCommManager(usbSerial, batteryData, wheelData));
+        super.onSerialReady(usbSerial);
+    }
+
+    @Override
+    public void onOutputsReady() {
+        publisherManager.initializePublishers();
+        publisherManager.startPublishers();
+    }
+
+    @Override
+    public void onWheelDataUpdate(long timestamp, int wheelCountL, int wheelCountR, double wheelDistanceL, double wheelDistanceR, double wheelSpeedInstantL, double wheelSpeedInstantR, double wheelSpeedBufferedL, double wheelSpeedBufferedR, double wheelSpeedExpAvgL, double wheelSpeedExpAvgR) {
+        countL = wheelCountL;
+        // TODO(kngwyu) why negated?
+        countR = -wheelCountR;
+        debugInfo.text2 = String.format("WheelCount: %d %d", countL, countR);
+    }
+
+    @Override
+    public void onBatteryVoltageUpdate(long timestamp, double voltage) {
+    }
+
+    @Override
+    public void onChargerVoltageUpdate(long timestamp, double chargerVoltage, double coilVoltage) {
+    }
+
+    @Override
+    public void onObjectsDetected(Bitmap bitmap, TensorImage tensorImage, List<Detection> results, long inferenceTime, int height, int width) {
+        try {
+            Matrix matrix = new Matrix();
+            matrix.postRotate(270);
+            Bitmap rotatedBitmap = Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true);
+            debugInfo.image = Bitmap.createScaledBitmap(rotatedBitmap, rotatedBitmap.getWidth() * 4, rotatedBitmap.getHeight() * 4, true);
+            Category category = results.get(0).getCategories().get(0);
+            imageLabel = category.getLabel();
+            debugInfo.text3 = String.format("Label: %s (score: %.2f)", imageLabel, category.getScore());
+        } catch (IndexOutOfBoundsException e) {
+            imageLabel = "Nothing";
+            debugInfo.text3 = "No object detected";
+        }
+    }
+}
diff --git a/apps/handsOnApp/src/main/res/drawable-v24/ic_launcher_foreground.xml b/apps/handsOnApp/src/main/res/drawable-v24/ic_launcher_foreground.xml
@@ -0,0 +1,34 @@
+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:aapt="http://schemas.android.com/aapt"
+    android:width="108dp"
+    android:height="108dp"
+    android:viewportWidth="108"
+    android:viewportHeight="108">
+    <path
+        android:fillType="evenOdd"
+        android:pathData="M32,64C32,64 38.39,52.99 44.13,50.95C51.37,48.37 70.14,49.57 70.14,49.57L108.26,87.69L108,109.01L75.97,107.97L32,64Z"
+        android:strokeWidth="1"
+        android:strokeColor="#00000000">
+        <aapt:attr name="android:fillColor">
+            <gradient
+                android:endX="78.5885"
+                android:endY="90.9159"
+                android:startX="48.7653"
+                android:startY="61.0927"
+                android:type="linear">
+                <item
+                    android:color="#44000000"
+                    android:offset="0.0" />
+                <item
+                    android:color="#00000000"
+                    android:offset="1.0" />
+            </gradient>
+        </aapt:attr>
+    </path>
+    <path
+        android:fillColor="#FFFFFF"
+        android:fillType="nonZero"
+        android:pathData="M66.94,46.02L66.94,46.02C72.44,50.07 76,56.61 76,64L32,64C32,56.61 35.56,50.11 40.98,46.06L36.18,41.19C35.45,40.45 35.45,39.3 36.18,38.56C36.91,37.81 38.05,37.81 38.78,38.56L44.25,44.05C47.18,42.57 50.48,41.71 54,41.71C57.48,41.71 60.78,42.57 63.68,44.05L69.11,38.56C69.84,37.81 70.98,37.81 71.71,38.56C72.44,39.3 72.44,40.45 71.71,41.19L66.94,46.02ZM62.94,56.92C64.08,56.92 65,56.01 65,54.88C65,53.76 64.08,52.85 62.94,52.85C61.8,52.85 60.88,53.76 60.88,54.88C60.88,56.01 61.8,56.92 62.94,56.92ZM45.06,56.92C46.2,56.92 47.13,56.01 47.13,54.88C47.13,53.76 46.2,52.85 45.06,52.85C43.92,52.85 43,53.76 43,54.88C43,56.01 43.92,56.92 45.06,56.92Z"
+        android:strokeWidth="1"
+        android:strokeColor="#00000000" />
+</vector>