Hello, my Name is Tim. I am attending Purdue University Indianapolis and I am currently on a senior Design team involving a UR5e Cobot. I am developing a URCap and I am having trouble getting the UI to update accordingly and I think I am having trouble getting this daemon server starting correctly. The goal of this URCap is to:
-
Create live camera feed using the usb camera plugged into the controller.
-
Detect 2D shapes or drawings shown to the camera
-
With the shapes it detects, draw with an expo marker (and an adjusted TCP to the tip of the expo marker) the shapes it sees on a vertically mounted white board.
-
The daemon should return movements that the Cobot can move using moveL or move J. (There is some backend calculations needing to take place to achieve this.
What I have achieve so far is a working python file. I was able to pip install numpy, and opencv using python 2.7. This is the version of python currently on the controller.
I need help making sure my code is correct and if there is any feedback or logic errors you guys see, this would be a big help. Thank you!
(As I am a new user, I cannot do file uploads. So here is some raw code)
package com.ECET.CameraDrawApp.impl;
import java.net.MalformedURLException;
import java.net.URL;
import com.ur.urcap.api.contribution.DaemonContribution;
import com.ur.urcap.api.contribution.DaemonService;
public class CameraDaemonService implements DaemonService{
private DaemonContribution daemonContribution;
@Override
public void init(DaemonContribution daemon) {
this.daemonContribution = daemon;
try {
// Ensure the daemon is installed correctly
daemonContribution.installResource(new URL("file:com/ECET/CameraDrawingApp/impl/daemon/"));
} catch (MalformedURLException e) {
System.err.println("Malformed URL in daemon service: " + e.getMessage());
} catch (Exception e) {
System.err.println("Failed to install daemon resource: " + e.getMessage());
}
}
@Override
public URL getExecutable() {
try {
return new URL("file:com/ECET/CameraDrawingApp/impl/daemon/camera-daemon.py");
} catch (MalformedURLException e) {
System.err.println("Could not load the daemon executable: " + e.getMessage());
return null;
}
}
public DaemonContribution getDaemon() {
return daemonContribution;
}
}
package com.ECET.CameraDrawApp.impl;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.imageio.ImageIO;
import org.apache.commons.codec.binary.Base64;
import org.apache.xmlrpc.XmlRpcException;
import org.apache.xmlrpc.client.XmlRpcClient;
import org.apache.xmlrpc.client.XmlRpcClientConfigImpl;
public class XmlRpcCameraDrawingAppDaemonInterface {
static final String SERVER_URL = “http://127.0.0.1:40405/RPC2”;
private final XmlRpcClient client;
private final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1);
private boolean daemonReachable;
public XmlRpcCameraDrawingAppDaemonInterface() throws Exception {
XmlRpcClientConfigImpl config = new XmlRpcClientConfigImpl();
config.setServerURL(new URL(SERVER_URL));
client = new XmlRpcClient();
client.setConfig(config);
startDaemonMonitor();
}
private void startDaemonMonitor() {
executorService.scheduleAtFixedRate(() -> {
try {
daemonReachable = ping();
} catch (Exception e) {
daemonReachable = false;
}
}, 0, 2, TimeUnit.SECONDS);
}
/**
* Checks if the daemon is reachable.
* @return True if daemon is reachable, false otherwise.
*/
public boolean isDaemonReachable() {
return daemonReachable;
}
/**
* Sends a ping request to the daemon.
* @return True if daemon responds, false otherwise.
*/
public boolean ping() {
try {
return (boolean) client.execute("ping", new Object[]{});
} catch (XmlRpcException e) {
return false;
}
}
/**
* Retrieves an image from the camera as a BufferedImage.
* The image is received as a Base64-encoded string and converted back to BufferedImage.
*/
public BufferedImage getCameraFrame() throws XmlRpcException {
try {
Object response = client.execute("GetImage", new ArrayList<>());
if (response instanceof String) {
byte[] imageBytes = Base64.decodeBase64((String) response);
return ImageIO.read(new ByteArrayInputStream(imageBytes));
} else {
throw new XmlRpcException("Invalid response type from daemon. Expected Base64 string.");
}
} catch (Exception e) {
throw new XmlRpcException("Error retrieving camera frame: " + e.getMessage());
}
}
/**
* Retrieves detected shapes from the camera feed.
* The data is structured as a list of coordinate points.
*/
public List<List<Double>> detectShapes() throws XmlRpcException {
try {
Object response = client.execute("detect_shapes", new ArrayList<>());
if (response instanceof Object[]) {
List<List<Double>> shapes = new ArrayList<>();
for (Object item : (Object[]) response) {
if (item instanceof List) {
List<?> rawList = (List<?>) item;
List<Double> shape = new ArrayList<>();
for (Object point : rawList) {
if (point instanceof Double) {
shape.add((Double) point);
} else {
throw new XmlRpcException("Invalid data type in shape coordinates.");
}
}
shapes.add(shape);
} else {
throw new XmlRpcException("Invalid shape data structure.");
}
}
return shapes;
} else {
throw new XmlRpcException("Invalid response type from daemon.");
}
} catch (Exception e) {
throw new XmlRpcException("Error retrieving detected shapes: " + e.getMessage());
}
}
/**
* Enables or disables auto-focus.
* @param enable True to enable auto-focus, False to disable.
*/
public void setAutoFocus(boolean enable) throws XmlRpcException {
try {
List<Object> params = new ArrayList<>();
params.add(enable);
client.execute("setEnableAutoFocus", params);
} catch (Exception e) {
throw new XmlRpcException("Error setting auto-focus: " + e.getMessage());
}
}
/**
* Sets the focus level of the camera.
* @param value Focus level (0-100).
*/
public void setFocusLevel(int value) throws XmlRpcException {
try {
List<Object> params = new ArrayList<>();
params.add(value);
client.execute("setFocusValue", params);
} catch (Exception e) {
throw new XmlRpcException("Error setting focus level: " + e.getMessage());
}
}
/**
* Sets the exposure level of the camera.
* @param value Exposure level (0-100).
*/
public void setExposureLevel(int value) throws XmlRpcException {
try {
List<Object> params = new ArrayList<>();
params.add(value);
client.execute("setExposureValue", params);
} catch (Exception e) {
throw new XmlRpcException("Error setting exposure level: " + e.getMessage());
}
}
public void stopMonitor() {
executorService.shutdown();
}
}
package com.ECET.CameraDrawApp.impl;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.swing.SwingUtilities;
import com.ur.urcap.api.contribution.InstallationNodeContribution;
import com.ur.urcap.api.contribution.installation.InstallationAPIProvider;
import com.ur.urcap.api.domain.data.DataModel;
import com.ur.urcap.api.domain.script.ScriptWriter;
public class CameraDrawingAppInstallationNodeContribution implements InstallationNodeContribution {
// Constants for DataModel keys
private static final String DAEMON_RUNNING_KEY = "daemonRunning";
private static final String AUTO_FOCUS_KEY = "autoFocus";
private static final String FOCUS_LEVEL_KEY = "focusLevel";
private static final String EXPOSURE_LEVEL_KEY = "exposureLevel";
// Dependencies
private final CameraDrawingAppInstallationNodeView view;
private final DataModel model;
private final CameraDaemonService daemonService;
private final XmlRpcCameraDrawingAppDaemonInterface daemonInterface;
// Executor for UI updates
private ScheduledExecutorService executorService;
public CameraDrawingAppInstallationNodeContribution(InstallationAPIProvider apiProvider,
CameraDrawingAppInstallationNodeView view,
DataModel model,
CameraDaemonService daemonService,
XmlRpcCameraDrawingAppDaemonInterface daemonInterface) {
this.view = view;
this.model = model;
this.daemonService = daemonService;
this.daemonInterface = daemonInterface;
this.executorService = Executors.newScheduledThreadPool(1);
initializeDataModel();
applyDaemonStatus();
}
private void initializeDataModel() {
if (!model.isSet(DAEMON_RUNNING_KEY)) model.set(DAEMON_RUNNING_KEY, false);
if (!model.isSet(AUTO_FOCUS_KEY)) model.set(AUTO_FOCUS_KEY, true);
if (!model.isSet(FOCUS_LEVEL_KEY)) model.set(FOCUS_LEVEL_KEY, 50);
if (!model.isSet(EXPOSURE_LEVEL_KEY)) model.set(EXPOSURE_LEVEL_KEY, 50);
}
@Override
public void openView() {
updateUI();
restartExecutorService();
}
@Override
public void closeView() {
shutdownExecutorService();
}
@Override
public void generateScript(ScriptWriter writer) {
writer.assign("camera", "rpc_factory(\"xmlrpc\", \" http://127.0.0.1:40405/RPC2\")");
writer.appendLine("if camera == None:");
writer.appendLine(" popup(\"Error: Unable to establish XML-RPC connection.\", \"Error\", False, True, False)");
writer.appendLine(" halt");
writer.appendLine("if camera.ping():");
writer.appendLine(" textmsg(\"Camera Daemon Connected\")");
writer.appendLine("else:");
writer.appendLine(" popup(\"Camera Daemon Unreachable\", \"Error\", False, True, False)");
}
public void onStartCameraPressed() {
model.set(DAEMON_RUNNING_KEY, true);
applyDaemonStatus();
updateUI();
}
public void onStopCameraPressed() {
model.set(DAEMON_RUNNING_KEY, false);
applyDaemonStatus();
updateUI();
}
public void toggleAutoFocus() {
boolean autoFocus = model.get(AUTO_FOCUS_KEY, true);
model.set(AUTO_FOCUS_KEY, !autoFocus);
updateUI();
executeAsync(() -> {
try {
daemonInterface.setAutoFocus(!autoFocus);
} catch (Exception e) {
logError("Failed to toggle auto-focus", e);
}
});
}
public void adjustFocus(int amount) {
int newFocus = clampValue(model.get(FOCUS_LEVEL_KEY, 50) + amount, 0, 100);
model.set(FOCUS_LEVEL_KEY, newFocus);
updateUI();
executeAsync(() -> {
try {
daemonInterface.setFocusLevel(newFocus);
} catch (Exception e) {
logError("Failed to adjust focus", e);
}
});
}
public void adjustExposure(int amount) {
int newExposure = clampValue(model.get(EXPOSURE_LEVEL_KEY, 50) + amount, 0, 100);
model.set(EXPOSURE_LEVEL_KEY, newExposure);
updateUI();
executeAsync(() -> {
try {
daemonInterface.setExposureLevel(newExposure);
} catch (Exception e) {
logError("Failed to adjust exposure", e);
}
});
}
private void applyDaemonStatus() {
executeAsync(() -> {
boolean shouldRun = model.get(DAEMON_RUNNING_KEY, false);
try {
if (shouldRun) {
if (daemonInterface.ping()) {
daemonService.getDaemon().start();
} else {
logError("Daemon is unreachable, not starting.", null);
}
} else {
daemonService.getDaemon().stop();
}
} catch (Exception e) {
logError("Failed to manage daemon status", e);
}
});
}
private void updateUI() {
boolean isRunning = model.get(DAEMON_RUNNING_KEY, false);
view.updateCameraStatus(isRunning ? "Camera Running" : "Camera Offline");
view.setStartButtonEnabled(!isRunning);
view.setStopButtonEnabled(isRunning);
}
private void restartExecutorService() {
if (executorService.isShutdown() || executorService.isTerminated()) {
executorService = Executors.newScheduledThreadPool(1);
}
executorService.scheduleAtFixedRate(() -> SwingUtilities.invokeLater(this::updateUI), 0, 2, TimeUnit.SECONDS);
}
private void shutdownExecutorService() {
if (executorService != null && !executorService.isShutdown()) {
executorService.shutdown();
}
}
private void executeAsync(Runnable task) {
new Thread(task).start();
}
private void logError(String message, Exception e) {
System.err.println(message + (e != null ? ": " + e.getMessage() : ""));
}
private int clampValue(int value, int min, int max) {
return Math.max(min, Math.min(max, value));
}
}
package com.ECET.CameraDrawApp.impl;
import java.awt.Color;
import java.awt.Component;
import java.awt.Dimension;
import java.awt.FlowLayout;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import javax.swing.BorderFactory;
import javax.swing.Box;
import javax.swing.BoxLayout;
import javax.swing.ImageIcon;
import javax.swing.JButton;
import javax.swing.JLabel;
import javax.swing.JPanel;
import com.ur.urcap.api.contribution.installation.swing.SwingInstallationNodeView;
public class CameraDrawingAppInstallationNodeView implements SwingInstallationNodeView{
private JButton startCameraButton;
private JButton stopCameraButton;
private JButton autoFocusButton;
private JButton increaseFocusButton;
private JButton decreaseFocusButton;
private JButton increaseExposureButton;
private JButton decreaseExposureButton;
private JLabel cameraStatusLabel;
private JLabel videoLabel;
private static final int VIDEO_WIDTH = 640;
private static final int VIDEO_HEIGHT = 480;
@Override
public void buildUI(JPanel panel, CameraDrawingAppInstallationNodeContribution contribution) {
panel.setLayout(new BoxLayout(panel, BoxLayout.Y_AXIS));
cameraStatusLabel = new JLabel("Status: Disconnected");
cameraStatusLabel.setAlignmentX(Component.LEFT_ALIGNMENT);
startCameraButton = new JButton("Start Camera");
stopCameraButton = new JButton("Stop Camera");
autoFocusButton = new JButton("Toggle Auto Focus");
increaseFocusButton = new JButton("Increase Focus");
decreaseFocusButton = new JButton("Decrease Focus");
increaseExposureButton = new JButton("Increase Exposure");
decreaseExposureButton = new JButton("Decrease Exposure");
videoLabel = new JLabel("No Video Available");
videoLabel.setPreferredSize(new Dimension(VIDEO_WIDTH, VIDEO_HEIGHT));
videoLabel.setMaximumSize(videoLabel.getPreferredSize());
videoLabel.setBorder(BorderFactory.createLineBorder(Color.BLACK));
startCameraButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
contribution.onStartCameraPressed();
}
});
stopCameraButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
contribution.onStopCameraPressed();
}
});
autoFocusButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
contribution.toggleAutoFocus();
}
});
increaseFocusButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
contribution.adjustFocus(1);
}
});
decreaseFocusButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
contribution.adjustFocus(-1);
}
});
increaseExposureButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
contribution.adjustExposure(1);
}
});
decreaseExposureButton.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent e) {
contribution.adjustExposure(-1);
}
});
// Layout Setup
panel.add(cameraStatusLabel);
panel.add(Box.createRigidArea(new Dimension(0, 10)));
panel.add(videoLabel);
panel.add(Box.createRigidArea(new Dimension(0, 10)));
JPanel buttonPanel = new JPanel();
buttonPanel.setLayout(new FlowLayout(FlowLayout.LEFT));
buttonPanel.add(startCameraButton);
buttonPanel.add(stopCameraButton);
buttonPanel.add(autoFocusButton);
panel.add(buttonPanel);
JPanel controlPanel = new JPanel();
controlPanel.setLayout(new FlowLayout(FlowLayout.LEFT));
controlPanel.add(increaseFocusButton);
controlPanel.add(decreaseFocusButton);
controlPanel.add(increaseExposureButton);
controlPanel.add(decreaseExposureButton);
panel.add(controlPanel);
}
public void updateCameraStatus(String status) {
cameraStatusLabel.setText(status);
}
public void updateVideoFeed(ImageIcon image) {
videoLabel.setIcon(image);
}
public void setStartButtonEnabled(boolean enabled) {
startCameraButton.setEnabled(enabled);
}
public void setStopButtonEnabled(boolean enabled) {
stopCameraButton.setEnabled(enabled);
}
}
Program node for drawing. In the program tree, it will just show the camera feed. At run, time if will popup and try to confirm the user has a drawing present.
package com.ECET.CameraDrawApp.impl;
import java.awt.image.BufferedImage;
import java.util.Timer;
import java.util.TimerTask;
import javax.swing.SwingUtilities;
import com.ur.urcap.api.contribution.ProgramNodeContribution;
import com.ur.urcap.api.contribution.program.ProgramAPIProvider;
import com.ur.urcap.api.domain.data.DataModel;
import com.ur.urcap.api.domain.script.ScriptWriter;
public class CameraDrawingAppProgramNodeContribution implements ProgramNodeContribution{
@SuppressWarnings("unused")
private final ProgramAPIProvider apiProvider;
private final CameraDrawingAppProgramNodeView view;
@SuppressWarnings("unused")
private final DataModel model;
private final XmlRpcCameraDrawingAppDaemonInterface daemonInterface;
private Timer uiTimer;
public CameraDrawingAppProgramNodeContribution(ProgramAPIProvider apiProvider,
CameraDrawingAppProgramNodeView view,
DataModel model,
XmlRpcCameraDrawingAppDaemonInterface daemonInterface) {
this.apiProvider = apiProvider;
this.view = view;
this.model = model;
this.daemonInterface = daemonInterface;
}
@Override
public void openView() {
if (uiTimer != null) {
uiTimer.cancel(); // Ensure no existing timer is running
}
uiTimer = new Timer(true);
uiTimer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
SwingUtilities.invokeLater(CameraDrawingAppProgramNodeContribution.this::updateUI);
}
}, 0, 1000); // Update every 1 second
}
@Override
public void closeView() {
if (uiTimer != null) {
uiTimer.cancel();
uiTimer = null;
}
}
@Override
public String getTitle() {
return "Camera Drawing";
}
@Override
public boolean isDefined() {
// TODO Auto-generated method stub
return true;
}
private void updateUI() {
try {
BufferedImage frame = daemonInterface.getCameraFrame(); // Now returns BufferedImage
if (frame != null) {
SwingUtilities.invokeLater(() -> view.updateVideoFeed(frame));
}
} catch (Exception e) {
System.err.println("Error fetching video frame: " + e.getMessage());
}
}
@Override
public void generateScript(ScriptWriter writer) {
writer.assign("cam", "rpc_factory(\"xmlrpc\", \"" + XmlRpcCameraDrawingAppDaemonInterface.SERVER_URL + "\")");
writer.appendLine("if cam.ping():");
writer.appendLine(" textmsg(\"Camera Daemon Connected\")");
writer.appendLine("else:");
writer.appendLine(" popup(\"Camera Daemon Unreachable\", \"Error\", False, True, False)");
writer.appendLine(" halt");
writer.appendLine("shapes = cam.detect_shapes()");
writer.appendLine("if str_len(to_str(shapes)) == 0 or shapes == None:");
writer.appendLine(" popup(\"No shapes detected!\", \"Error\", False, True, False)");
writer.appendLine(" halt");
writer.appendLine("board_origin = p[0.5, 0.2, 0.1, 0, 3.1415, 0]");
writer.appendLine("approach_height = 0.05");
writer.appendLine("drawing_speed = 0.1");
writer.appendLine("move_speed = 0.25");
writer.appendLine("shape_idx = 0");
writer.appendLine("while shape_idx < length(shapes):");
writer.appendLine(" shape = shapes[shape_idx]");
writer.appendLine(" if length(shape) > 0:");
writer.appendLine(" pt_idx = 0");
writer.appendLine(" pt0 = shape[pt_idx]");
writer.appendLine(" target0 = pose_trans(board_origin, p[pt0[0]*0.0254, pt0[1]*0.0254, 0, 0, 0, 0])");
writer.appendLine(" movej(pose_trans(target0, p[0,0,approach_height,0,0,0]), a=1.2, v=move_speed)");
writer.appendLine(" movel(target0, a=0.5, v=drawing_speed)");
writer.appendLine(" pt_idx = pt_idx + 1");
writer.appendLine(" while pt_idx < length(shape):");
writer.appendLine(" pt = shape[pt_idx]");
writer.appendLine(" target_pt = pose_trans(board_origin, p[pt[0]*0.0254, pt[1]*0.0254, 0, 0, 0, 0])");
writer.appendLine(" movel(target_pt, a=0.5, v=drawing_speed)");
writer.appendLine(" pt_idx = pt_idx + 1");
writer.appendLine(" end");
writer.appendLine(" movel(target0, a=0.5, v=drawing_speed)");
writer.appendLine(" movej(pose_trans(target0, p[0,0,approach_height,0,0,0]), a=1.2, v=move_speed)");
writer.appendLine(" end");
writer.appendLine(" shape_idx = shape_idx + 1");
writer.appendLine("end");
writer.appendLine("popup(\"Shape drawing completed!\", \"Done\", False, False, False)");
}
}
package com.ECET.CameraDrawApp.impl;
import java.awt.Color;
import java.awt.Dimension;
import java.awt.image.BufferedImage;
import javax.swing.BorderFactory;
import javax.swing.BoxLayout;
import javax.swing.ImageIcon;
import javax.swing.JLabel;
import javax.swing.JPanel;
import com.ur.urcap.api.contribution.ContributionProvider;
import com.ur.urcap.api.contribution.program.swing.SwingProgramNodeView;
public class CameraDrawingAppProgramNodeView implements SwingProgramNodeView{
private JLabel videoLabel;
private static final int VIDEO_WIDTH = 640;
private static final int VIDEO_HEIGHT = 480;
@Override
public void buildUI(JPanel panel, ContributionProvider<CameraDrawingAppProgramNodeContribution> provider) {
panel.setLayout(new BoxLayout(panel, BoxLayout.Y_AXIS));
videoLabel = new JLabel("No Video Feed");
videoLabel.setPreferredSize(new Dimension(VIDEO_WIDTH, VIDEO_HEIGHT));
videoLabel.setMaximumSize(videoLabel.getPreferredSize());
videoLabel.setBorder(BorderFactory.createLineBorder(Color.BLACK));
panel.add(videoLabel);
}
/**
* Updates the video feed with a BufferedImage.
* @param image The BufferedImage to display.
*/
public void updateVideoFeed(BufferedImage image) {
if (image != null) {
videoLabel.setIcon(new ImageIcon(image));
} else {
videoLabel.setText("No Video Available");
}
}
}
If this is too much to go into detail in this discussion, I have the GitHub link below.