I am trying to build a real-time object detection android app using the Google ML kit. What I want to do is take a image frame from the camera , detect the objects inside that frame , draw the bounding boxes & then show that image to the user on the screen. I am trying to do this using the previewView, how can I process the images before the previewView shows them to the user? Is this even possible ? Please help me out here. Thanks
activity_main.xml
<?xml version="1.0" encoding="utf-8"?>
<androidx.constraintlayout.widget.ConstraintLayout
xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
tools:context=".MainActivity">
<androidx.appcompat.widget.Toolbar
android:id="@+id/actionBar_main"
android:layout_width="match_parent"
android:layout_height="?attr/actionBarSize"
android:background="@color/teal_200"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toTopOf="parent"
app:title="@string/app_name" />
<androidx.camera.view.PreviewView
android:id="@+id/previewView_main"
android:layout_width="match_parent"
android:layout_height="0dp"
app:layout_constraintBottom_toBottomOf="parent"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@+id/actionBar_main" />
</androidx.constraintlayout.widget.ConstraintLayout>
MainActivity.java
import androidx.core.content.ContextCompat;
import androidx.lifecycle.LifecycleOwner;
import android.Manifest;
import android.annotation.SuppressLint;
import android.content.pm.PackageManager;
import android.media.Image;
import android.os.Bundle;
import android.util.Size;
import com.google.android.gms.tasks.OnFailureListener;
import com.google.android.gms.tasks.OnSuccessListener;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.mlkit.vision.common.InputImage;
import com.google.mlkit.vision.objects.DetectedObject;
import com.google.mlkit.vision.objects.ObjectDetection;
import com.google.mlkit.vision.objects.ObjectDetector;
import com.google.mlkit.vision.objects.defaults.ObjectDetectorOptions;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
public class MainActivity extends AppCompatActivity {
Toolbar toolbar;
PreviewView previewView_main;
ListenableFuture<ProcessCameraProvider> cameraProviderFuture;
private final int CAMERA_REQUEST_CODE = 101;
ObjectDetectorOptions options;
ExecutorService cameraExecutor;
ObjectDetector objectDetector;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
cameraExecutor = Executors.newSingleThreadExecutor();
toolbar = findViewById(R.id.actionBar_main);
setSupportActionBar(toolbar);
previewView_main = findViewById(R.id.previewView_main);
ask_permissions();
options = new ObjectDetectorOptions.Builder()
.setDetectorMode(ObjectDetectorOptions.STREAM_MODE)
.enableClassification()
.build();
objectDetector = ObjectDetection.getClient(options);
}
private void load_camera_preview() {
cameraProviderFuture = ProcessCameraProvider.getInstance(this);
cameraProviderFuture.addListener(() -> {
try {
ProcessCameraProvider cameraProvider = cameraProviderFuture.get();
bindPreview(cameraProvider);
} catch (ExecutionException | InterruptedException e) {
}
}, ContextCompat.getMainExecutor(this));
}
private void bindPreview(ProcessCameraProvider cameraProvider) {
Preview preview = new Preview.Builder()
.build();
CameraSelector cameraSelector = new CameraSelector.Builder()
.requireLensFacing(CameraSelector.LENS_FACING_BACK)
.build();
preview.setSurfaceProvider(previewView_main.getSurfaceProvider());
ImageAnalysis imageAnalysis =
new ImageAnalysis.Builder()
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
.setTargetResolution(new Size(1280, 720))
.build();
imageAnalysis.setAnalyzer(cameraExecutor, new ImageAnalysis.Analyzer() {
@Override
public void analyze(@NonNull ImageProxy image) {
@SuppressLint("UnsafeOptInUsageError")
Image mediaImage = image.getImage();
if (mediaImage != null) {
InputImage inputImage = InputImage.fromMediaImage
(mediaImage, image.getImageInfo().getRotationDegrees());
objectDetector.process(inputImage)
.addOnSuccessListener(detectedObjects -> {
})
.addOnFailureListener(e -> {
});
}
image.close();
}
});
Camera camera = cameraProvider.bindToLifecycle((LifecycleOwner) this, cameraSelector, preview);
}
private void ask_permissions() {
if (ActivityCompat.checkSelfPermission(this, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
String[] permissions = {Manifest.permission.CAMERA};
ActivityCompat.requestPermissions(this, permissions, CAMERA_REQUEST_CODE);
}
if (ActivityCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED) {
load_camera_preview();
}
}
@Override
public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
if (requestCode == CAMERA_REQUEST_CODE && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
load_camera_preview();
} else {
load_camera_preview();
}
}
What I have tried:
I was thinking if I could overlay an imageview over the entire previewView and then call change the images inside the imageview, but that doesnt seem efficient.