/* * Copyright 2017 The Android Things Samples Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.example.androidthings.imageclassifier; import android.app.Activity; import android.content.pm.PackageManager; import android.graphics.Bitmap; import android.media.Image; import android.media.ImageReader; import android.os.Build; import android.os.Bundle; import android.os.Handler; import android.os.HandlerThread; import android.speech.tts.TextToSpeech; import android.speech.tts.UtteranceProgressListener; import android.support.annotation.NonNull; import android.util.Log; import android.view.KeyEvent; import android.view.WindowManager; import android.widget.ImageView; import android.widget.TextView; import android.widget.Toast; import com.example.androidthings.imageclassifier.classifier.Classifier; import com.example.androidthings.imageclassifier.classifier.TensorFlowImageClassifier; import com.google.android.things.contrib.driver.button.Button; import com.google.android.things.contrib.driver.button.ButtonInputDriver; import com.google.android.things.pio.Gpio; import com.google.android.things.pio.PeripheralManagerService; import java.io.IOException; import java.util.List; import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; import static android.Manifest.permission.CAMERA; import static android.Manifest.permission.WRITE_EXTERNAL_STORAGE; public class ImageClassifierActivity extends Activity implements ImageReader.OnImageAvailableListener { private static final String TAG = "ImageClassifierActivity"; private static final int PERMISSIONS_REQUEST = 1; private static final String BUTTON_PIN = "BCM21"; private static final String LED_PIN = "BCM6"; private ImagePreprocessor mImagePreprocessor; private TextToSpeech mTtsEngine; private TtsSpeaker mTtsSpeaker; private CameraHandler mCameraHandler; private TensorFlowImageClassifier mTensorFlowClassifier; private HandlerThread mBackgroundThread; private Handler mBackgroundHandler; private ImageView mImage; private TextView[] mResultViews; private AtomicBoolean mReady = new AtomicBoolean(false); private ButtonInputDriver mButtonDriver; private Gpio mReadyLED; @Override protected void onCreate(final Bundle savedInstanceState) { super.onCreate(savedInstanceState); getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); setContentView(R.layout.activity_camera); mImage = (ImageView) findViewById(R.id.imageView); mResultViews = new TextView[3]; mResultViews[0] = (TextView) findViewById(R.id.result1); mResultViews[1] = (TextView) findViewById(R.id.result2); mResultViews[2] = (TextView) findViewById(R.id.result3); if (hasPermission()) { if (savedInstanceState == null) { init(); } } else { requestPermission(); } } private void init() { initPIO(); mBackgroundThread = new HandlerThread("BackgroundThread"); mBackgroundThread.start(); mBackgroundHandler = new Handler(mBackgroundThread.getLooper()); mBackgroundHandler.post(mInitializeOnBackground); } private void initPIO() { PeripheralManagerService service = new PeripheralManagerService(); List<String> gpioList = service.getGpioList(); if (gpioList.contains(BUTTON_PIN)) { try { mButtonDriver = new ButtonInputDriver(BUTTON_PIN, Button.LogicState.PRESSED_WHEN_LOW, KeyEvent.KEYCODE_ENTER); mButtonDriver.register(); } catch (IOException e) { mButtonDriver = null; Log.w(TAG, "Could not open GPIO", e); } } if (mButtonDriver == null) { Log.w(TAG, "Cannot find pin " + BUTTON_PIN + ". Ignoring push button on PIO. " + "Use a keyboard instead"); } if (gpioList.contains(LED_PIN)) { try { mReadyLED = service.openGpio(LED_PIN); mReadyLED.setDirection(Gpio.DIRECTION_OUT_INITIALLY_LOW); } catch (IOException e) { mReadyLED = null; Log.w(TAG, "Could not open GPIO", e); } } if (mReadyLED == null) { Log.w(TAG, "Cannot find pin " + LED_PIN + ". Ignoring ready indicator LED."); } } private Runnable mInitializeOnBackground = new Runnable() { @Override public void run() { mImagePreprocessor = new ImagePreprocessor(CameraHandler.IMAGE_WIDTH, CameraHandler.IMAGE_HEIGHT, TensorFlowImageClassifier.INPUT_SIZE); mTtsSpeaker = new TtsSpeaker(); mTtsSpeaker.setHasSenseOfHumor(true); mTtsEngine = new TextToSpeech(ImageClassifierActivity.this, new TextToSpeech.OnInitListener() { @Override public void onInit(int status) { if (status == TextToSpeech.SUCCESS) { mTtsEngine.setLanguage(Locale.US); mTtsEngine.setOnUtteranceProgressListener(utteranceListener); mTtsSpeaker.speakReady(mTtsEngine); } else { Log.w(TAG, "Could not open TTS Engine (onInit status=" + status + "). Ignoring text to speech"); mTtsEngine = null; } } }); mCameraHandler = CameraHandler.getInstance(); mCameraHandler.initializeCamera( ImageClassifierActivity.this, mBackgroundHandler, ImageClassifierActivity.this); mTensorFlowClassifier = new TensorFlowImageClassifier(ImageClassifierActivity.this); setReady(true); } }; private Runnable mBackgroundClickHandler = new Runnable() { @Override public void run() { if (mTtsEngine != null) { mTtsSpeaker.speakShutterSound(mTtsEngine); } mCameraHandler.takePicture(); } }; private UtteranceProgressListener utteranceListener = new UtteranceProgressListener() { @Override public void onStart(String utteranceId) { setReady(false); } @Override public void onDone(String utteranceId) { setReady(true); } @Override public void onError(String utteranceId) { setReady(true); } }; @Override public boolean onKeyDown(int keyCode, KeyEvent event) { Log.d(TAG, "Received key down: " + keyCode + ". Ready = " + mReady.get()); if (keyCode == KeyEvent.KEYCODE_ENTER || keyCode == KeyEvent.KEYCODE_VOLUME_DOWN) { if (mReady.get()) { setReady(false); mBackgroundHandler.post(mBackgroundClickHandler); } else { Log.i(TAG, "Sorry, processing hasn't finished. Try again in a few seconds"); } return true; } return super.onKeyDown(keyCode, event); } private void setReady(boolean ready) { mReady.set(ready); if (mReadyLED != null) { try { mReadyLED.setValue(ready); } catch (IOException e) { Log.w(TAG, "Could not set LED", e); } } } @Override public void onImageAvailable(ImageReader reader) { final Bitmap bitmap; try (Image image = reader.acquireNextImage()) { bitmap = mImagePreprocessor.preprocessImage(image); } runOnUiThread(new Runnable() { @Override public void run() { mImage.setImageBitmap(bitmap); } }); final List<Classifier.Recognition> results = mTensorFlowClassifier.recognizeImage(bitmap); Log.d(TAG, "Got the following results from Tensorflow: " + results); if (mTtsEngine != null) { // speak out loud the result of the image recognition mTtsSpeaker.speakResults(mTtsEngine, results); } else { // if theres no TTS, we don't need to wait until the utterance is spoken, so we set // to ready right away. setReady(true); } runOnUiThread(new Runnable() { @Override public void run() { for (int i = 0; i < mResultViews.length; i++) { if (results.size() > i) { Classifier.Recognition r = results.get(i); mResultViews[i].setText(r.toString()); } else { mResultViews[i].setText(null); } } } }); } @Override protected void onDestroy() { super.onDestroy(); try { if (mBackgroundThread != null) mBackgroundThread.quit(); } catch (Throwable t) { // close quietly } mBackgroundThread = null; mBackgroundHandler = null; try { if (mCameraHandler != null) mCameraHandler.shutDown(); } catch (Throwable t) { // close quietly } try { if (mTensorFlowClassifier != null) mTensorFlowClassifier.close(); } catch (Throwable t) { // close quietly } try { if (mButtonDriver != null) mButtonDriver.close(); } catch (Throwable t) { // close quietly } if (mTtsEngine != null) { mTtsEngine.stop(); mTtsEngine.shutdown(); } } // Permission-related methods. This is not needed for Android Things, where permissions are // automatically granted. However, it is kept here in case the developer needs to test on a // regular Android device. @Override public void onRequestPermissionsResult(int requestCode, @NonNull String permissions[], @NonNull int[] grantResults) { switch (requestCode) { case PERMISSIONS_REQUEST: { if (grantResults.length > 0 && grantResults[0] == PackageManager.PERMISSION_GRANTED && grantResults[1] == PackageManager.PERMISSION_GRANTED) { init(); } else { requestPermission(); } } } } private boolean hasPermission() { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { return true; } return checkSelfPermission(CAMERA) == PackageManager.PERMISSION_GRANTED && checkSelfPermission(WRITE_EXTERNAL_STORAGE) == PackageManager.PERMISSION_GRANTED; } private void requestPermission() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { if (shouldShowRequestPermissionRationale(CAMERA) || shouldShowRequestPermissionRationale(WRITE_EXTERNAL_STORAGE)) { Toast.makeText(this, "Camera AND storage permission are required for this demo", Toast.LENGTH_LONG).show(); } requestPermissions(new String[]{CAMERA, WRITE_EXTERNAL_STORAGE}, PERMISSIONS_REQUEST); } } }