.idea/compiler.xml

@@ -6,8 +6,16 @@
         <sourceOutputDir name="target/generated-sources/annotations" />
         <sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
         <outputRelativeToContentRoot value="true" />
-        <module name="group8" />
+        <module name="vslam-objects" />
       </profile>
     </annotationProcessing>
+    <bytecodeTargetLevel>
+      <module name="group8" target="17" />
+    </bytecodeTargetLevel>
+  </component>
+  <component name="JavacSettings">
+    <option name="ADDITIONAL_OPTIONS_OVERRIDE">
+      <module name="vslam-objects" options="-parameters" />
+    </option>
   </component>
 </project>
\ No newline at end of file

.idea/misc.xml

@@ -9,6 +9,10 @@
     </option>
     <option name="workspaceImportForciblyTurnedOn" value="true" />
   </component>
+  <component name="PWA">
+    <option name="enabled" value="true" />
+    <option name="wasEnabledAtLeastOnce" value="true" />
+  </component>
   <component name="ProjectRootManager" version="2" languageLevel="JDK_17" default="true" project-jdk-name="17" project-jdk-type="JavaSDK">
     <output url="file://$PROJECT_DIR$/out" />
   </component>

INSTALL.txt

+INSTALL.txt
+===========
+
+Pre-conditions:
+---------------
+This code requires a Unix/Linux-based system due to the shell script dependencies and environment configurations. It has been specifically tested on standard lab machines but should be compatible with most Linux distributions. The software relies on Java 17 and Apache Maven, which are included in the installation steps.
+
+Hardware Requirements:
+- A machine capable of running Java 17 efficiently, preferably with at least 4GB of RAM for optimal performance of the object detection and VSLAM processing.
+
+Software Requirements:
+- Java JDK 17
+- Apache Maven 3.9.6
+- A Unix/Linux-based operating system
+
+Supporting Files:
+-----------------
+All required libraries are managed through Maven, which will handle the dependencies automatically as specified in the project's pom.xml file. No additional manual installation of libraries is required beyond the initial setup performed by `install.sh`.
+
+Examples of Project Usage:
+
+- `data_installation.sh`: Download and install one of two datasets based on the user's input. The available options are:
+    1. TUM_RGB: The Freiburg3 Long Office Household dataset, suitable for testing robustness in indoor environments.
+    2. EuRoC_Mav: The Machine Hall 01 Easy dataset from the EuRoC MAV dataset series, ideal for aerial vehicle-based environments.
+
+    Usage Example:
+    `./data_installation.sh TUM_RGB`
+    `./data_installation.sh EuRoC_Mav`
+
+    *NOTE TO PEER REVIEWERS: RUNNING THIS SCIPT IS NOT NECESSARY FOR ASSESSMENT OF OUR PROJECT. We provided KeyFrames to execute the main components already*
+
+- `run_obj_detection.sh`: Perform object detection on generated key frames, modifying the project configuration temporarily to use YOLO.
+
+    To measure effectiveness of this script, remove the following two provided directories `/src/main/java/vslam/BoundedInfo` and `/src/main/java/vslam/BoundedKeyFrames`
+    Execution of this script should regenerate those two directories
+
+    *NOTE TO PEER REVIEWERS: RUNNING THIS SCIPT IS NOT NEEDED TO EXECUTE OUR PROJECT. THE BoundedInfo is pregenerated for your convinience*
+
+- `data_installation.sh` installs a large amount of data and `run_vslam.sh` executes Matlab code under the hood, so they are not required to be run by peer reviewers
+
+Descriptions of Testing Patterns:
+---------------------------------
+Testing in this project is divided into two main categories:
+1. Object Detection Tests:
+   - Once Maven is succesfully installed, you can execute the testing scripts by opening a terminal in the 'group8' directory and executing the 'mvn test' command. After the test completion, you can verify results of the unit tests. The tests focus on edge cases, precision, and equality for the 'Point', 'ObjectSet', and 'BoundingBox' types.
+
+2. YOLO Net Tests:
+    - Within the 'group8' directory, navigate to 'src/test/java/yolo'. Note that 'TestFrame.png' has no bounding boxes around the objects.
+
+    - Open a new terminal and run the 'mvn exec:java -Dexec.mainClass="yolo.YOLOTest" -Dexec.classpathScope="test"' command. Wait for full completion and an extra minute to generate files.
+
+    - To verify the functionality of YOLO Net, check that the 'TestFrame.png' image now contains bounding boxes. There will also be a new 'TestFrame.csv' file created in the current directory; for further verification, check that the values in that file are identical to those in the 'KeyFrame_0060.csv' file containing the ground truth values.
+
+Execution:
+----------
+To compile and run the project:
+1. Execute `./install.sh`
+2. Run `source ~/.bashrc`
+3. Execute `mvn clean install -DskipTests` to build the project, skipping tests during development to speed up the build process.
+4. Run `mvn exec:java` to execute the main application, which will start the configured object detection or VSLAM process as specified in the pom.xml file's main class configuration.
+5. Open up chrome and go to http://localhost:5555/ 
+6. You should be able to see objects detected using yolo and tracked in 3d space using our algorithm. Select various objects and see them represnted in the 3D point map
+
+This documentation ensures that any student or developer can set up, understand, and use the project from scratch on a compatible machine.

UpdateEdits.md

+### Accepted feedback:
+- "Enhancement in the visual interface" by Manuel Segimon
+    - "I suggest a potential enhancement in the visual interface. Instead of adding a new object below the current one, consider a design that deletes the current one and adds a new one. This way, the page doesn’t get bigger the more objects you look at, improving the user experience."
+        - Our front end now should be able to do just that
+
+- "Confusing object selection in browser" by Seyed Reza Sajjadinasab
+    - "It is not clear what are the different 0-14 objects. Selecting between them didn't result in any change in the output."
+        - We are adding class display for object numbers to the front end
+
+- "install.txt is misleading" by Seyed Reza Sajjadinasab
+    - "It is said that the instruction provided is for "A Unix/Linux-based operating system", however, the installation source is a java library from a location in the lab machines. Using it on another Linux system will change "~/.bashrc" and cause problems for further usage of the OS.
+    The received error:
+    -bash: /ad/eng/opt/java/add_jdk17.sh: No such file or directory"
+        - This should be addressed now in the install.txt
+
+- "Ask user what type of object to detect" by Seyed Reza Sajjadinasab
+    - "You can specify a few types of objects that are detectable. It is not clear now what type of object is detected running mvn exec:java -Dexec.mainClass="yolo.YOLOTest" -Dexec.classpathScope="test". For example, why books are not detected as objects?" 
+        - The class names next to object names should clear this up
+
+### Rejected feedback:
+- "Web Interface improvement" by Tejas Thakur Singh
+    - "Add tabs or a compact grid view in order to fix the organization issues that come with having multiple views open"
+        - Sort of addressed already by Manuel's feedback
+
+- "Java implementation of the GUI" by Seyed Reza Sajjadinasab
+    - "It was required to do the majority of the project with Java. I think it's better to provide a Java version for the GUI as well instead of java script GUI."
+        - We focused on functionality of the GUI for the purposes of our 3D localization, and javascript is better suited with the necessary libraries to perform this task
+
+- "Showing the original video/picture" by Seyed Reza Sajjadinasab
+    - "The original video can be shown in the browser to make it easier to identify the significance of the tool and the process."
+        - Our reconstructed dense object map will take this into account. Furthermore, we are going to show the original scenario(s) the VSLAM is performed on during our demo. The final output on the front end is color mapped, so the objects are reletively visible in a 3D display of the scene
+
+- "Window size not adjusted" by Seyed Reza Sajjadinasab
+    - "The size of the window is not adjusted while using a browser to view the result."
+        - This seemed to be a sececondary issue to our main goal for the project
+
+- "Command line user interface for image and video" by Seyed Reza Sajjadinasab
+    - "It will be good if the user can pass the filename to the tool and it does the detection on the specified file."
+        - We have scripts available to perform this task on the backend, and we provide 2 RGBD datasets where users can perform VSLAM on for our functional front end. We also provide video captured using a monocular camera source that we can perform VSLAM on, but this dataset is not displayed in the front end due to our algorithms incompatibility with depth-less data
+
+### Additional changes
+- Improved object tracking algorithm
+- Improved VSLAM by incorportaing VSLAM RGBD
+- Ability to perform monocularr VSLAM on footage collected from arbitrary data source
+- The biggest issue that was raised was the GUI, due to its lack of clarity. We have completely overhauled the frontend, and improved the backend to provide better accuracy. The following list details how we did this:
+    1. Migrated from the THREE.js pointcloud plotting library to Plotly.js
+    2. Improved styling of the frontend

benchmark_yolo.txt

+Total processing time: 80972 ms for 87 frames.
+Average time per frame: 915 ms.

data_installation.sh

+#!/bin/bash
+
+# Define the target directory for dataset installation
+TARGET_DIR="/src/main/java/vslam"
+
+# Check for correct usage
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 {TUM_RGB|EuRoC_Mav}"
+    exit 1
+fi
+
+# Setup dataset URLs and filenames based on the argument
+if [ "$1" = "TUM_RGB" ]; then
+    URL="https://cvg.cit.tum.de/rgbd/dataset/freiburg3/rgbd_dataset_freiburg3_long_office_household.tgz"
+    FILENAME="rgbd_dataset_freiburg3_long_office_household.tgz"
+elif [ "$1" = "EuRoC_Mav" ]; then
+    URL="http://robotics.ethz.ch/~asl-datasets/ijrr_euroc_mav_dataset/machine_hall/MH_01_easy/MH_01_easy.zip"
+    FILENAME="MH_01_easy.zip"
+else
+    echo "Invalid dataset name. Choose either TUM_RGB or EuRoC_Mav."
+    exit 2
+fi
+
+# Create target directory if it does not exist
+mkdir -p $TARGET_DIR
+
+# Change to the target directory
+cd $TARGET_DIR
+
+# Download the dataset
+echo "Downloading $FILENAME ..."
+wget $URL -O $FILENAME
+
+# Extract the dataset
+echo "Extracting $FILENAME ..."
+if [[ $FILENAME == *.tgz ]]; then
+    tar -xzvf $FILENAME
+elif [[ $FILENAME == *.zip ]]; then
+    unzip $FILENAME
+fi
+
+# Remove the compressed file after extraction
+rm $FILENAME
+
+echo "Installation and extraction complete!"
+

install.sh

+#!/bin/bash
+
+# Download Apache Maven
+echo "Downloading Maven..."
+wget https://dlcdn.apache.org/maven/maven-3/3.9.6/binaries/apache-maven-3.9.6-bin.tar.gz
+
+# Extract the downloaded tarball
+echo "Extracting Maven..."
+tar -xvf apache-maven-3.9.6-bin.tar.gz
+
+# Move Maven to the target directory
+echo "Moving Maven to /tmp/"
+mv apache-maven-3.9.6 /tmp/
+
+# Set M2_HOME and update PATH
+M2_HOME="/tmp/apache-maven-3.9.6"
+PATH_UPDATE="/tmp/apache-maven-3.9.6/bin:\$PATH"
+
+# Remove tarball file
+rm -rf apache-maven-3.9.6-bin.tar.gz
+
+echo "Installation complete."
+
+# Append M2_HOME and PATH to the user's .bashrc (or other appropriate file)
+echo "Updating environment variables..."
+echo "export M2_HOME=$M2_HOME" >> ~/.bashrc
+echo "export PATH=$PATH_UPDATE" >> ~/.bashrc
+
+# Optional: Automatically source Java upgrade script
+echo "source /ad/eng/opt/java/add_jdk17.sh" >> ~/.bashrc
+
+echo "Please close and reopen your terminal or run 'source ~/.bashrc' to apply all changes."
+echo "Please verify Java version with 'java -version'"
+echo "Please verify the Maven installation with: 'mvn -version'"
+

pom.xml

@@ -8,58 +8,72 @@
     <artifactId>vslam-objects</artifactId>
     <version>1.0</version>
 
-    <properties>
+     <properties>
         <maven.compiler.source>17</maven.compiler.source>
         <maven.compiler.target>17</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <exec.mainClass>EntryPoint</exec.mainClass>
+        <exec.mainClass>top.BackendJava</exec.mainClass>
     </properties>
 
-    <!-- FOR THE YOLO USAGE:
-    <properties>
-        <maven.compiler.source>17</maven.compiler.source>
-        <maven.compiler.target>17</maven.compiler.target>
-        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <exec.mainClass>yolo.YOLODetector</exec.mainClass>
-    </properties>
-    -->
-
-
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.0</version> <!-- Try changing the version here -->
+                <configuration>
+                    <source>17</source>
+                    <target>17</target>
+                    <compilerArgs>
+                        <arg>-parameters</arg>
+                    </compilerArgs>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
     <dependencies>
-        <!-- Spark framework for webdev -->
+        <!-- Spring Boot Starter Web -->
         <dependency>
-            <groupId>com.sparkjava</groupId>
-            <artifactId>spark-core</artifactId>
-            <version>2.9.4</version>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+            <version>3.2.4</version>
         </dependency>
+
+        <!-- Spring Boot Starter Thymeleaf -->
         <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-api</artifactId>
-            <version>2.20.0</version>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-thymeleaf</artifactId>
+            <version>3.2.4</version>
         </dependency>
+
+        <!-- Spring Boot DevTools -->
         <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-core</artifactId>
-            <version>2.20.0</version>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-devtools</artifactId>
+            <version>3.2.4</version>
         </dependency>
+
+        <!-- Spring Web MVC -->
         <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-slf4j-impl</artifactId>
-            <version>2.20.0</version>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-webmvc</artifactId>
+            <version>6.1.4</version>
         </dependency>
-        <!-- For JUnit (testing), with JUnit Jupiter -->
+
+        <!-- Gson for JSON processing -->
         <dependency>
-            <groupId>org.junit.jupiter</groupId>
-            <artifactId>junit-jupiter-api</artifactId>
-            <version>5.7.0</version>
-            <scope>test</scope>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.8.9</version>
         </dependency>
-        <!-- MongoDB -->
+
+        <!-- MongoDB Driver -->
         <dependency>
             <groupId>org.mongodb</groupId>
             <artifactId>mongodb-driver-sync</artifactId>
-            <version>5.0.1</version>
+            <version>4.7.1</version>
         </dependency>
+
         <!-- JavaCV dependencies -->
         <dependency>
             <groupId>org.bytedeco</groupId>
@@ -81,6 +95,34 @@
             <artifactId>javacpp</artifactId>
             <version>1.5.7</version>
         </dependency>
+        <dependency>
+            <groupId>com.opencsv</groupId>
+            <artifactId>opencsv</artifactId>
+            <version>5.7.0</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.ejml/ejml-all -->
+        <dependency>
+            <groupId>org.ejml</groupId>
+            <artifactId>ejml-all</artifactId>
+            <version>0.41</version>
+        </dependency>
+
+
+
+
+        <!-- Spring Boot Starter Test -->
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-test</artifactId>
+            <version>3.2.4</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.junit.v4</groupId>
+                    <artifactId>junit</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
     </dependencies>
-    
-</project>
+</project>
\ No newline at end of file

run_object_detection.sh

+#!/bin/bash
+
+# Define the POM file location
+POM_FILE="pom.xml"
+
+# Function to modify the POM file to use YOLODetector
+use_yolo_detector() {
+    # Backup the original POM file
+    cp $POM_FILE "${POM_FILE}.bak"
+
+    # Replace the main class to use YOLODetector
+    sed -i 's\<exec.mainClass>top.BackendJava</exec.mainClass>\<exec.mainClass>yolo.YOLODetector</exec.mainClass>\g' $POM_FILE
+
+}
+
+# Function to restore the original POM file
+restore_original_pom() {
+    # Restore the original POM file from backup
+    mv "${POM_FILE}.bak" $POM_FILE
+}
+
+# Use YOLODetector for object detection
+use_yolo_detector
+
+# Run Maven commands to clean, build, and execute the project
+mvn clean install -DskipTests
+mvn exec:java
+
+# Restore the original POM configuration
+restore_original_pom
+
+echo "Object detection has completed. The POM file has been restored to its original configuration."

run_vslam_mono.sh

+#!/bin/bash
+
+# Define the script that runs the VSLAM implementation
+<<<<<<< HEAD
+<<<<<<< HEAD
+MATLAB_SCRIPT="/src/main/java/vslam/vslam_implementation_rgbd.m"
+
+# Check if the correct number of arguments was provided
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 {tum_rgbd_dataset|imperial_college_london}"
+    exit 1
+fi
+
+# Determine which dataset to use based on the argument provided
+if [ "$1" = "tum_rgbd_dataset" ]; then
+    DATASET_NAME="tum_rgbd_dataset"
+elif [ "$1" = "imperial_college_london" ]; then
+    DATASET_NAME="imperial_college_london"
+else
+    echo "Invalid dataset name. Choose either 'tum_rgbd_dataset' or 'imperial_college_london'"
+    exit 2
+fi
+=======
+MATLAB_SCRIPT="/src/main/java/vslam/vslam_implementation.m"
+
+# Define the dataset name
+DATASET_NAME="self_made_dataset"
+>>>>>>> divider
+=======
+MATLAB_SCRIPT="/src/main/java/vslam/vslam_implementation.m"
+
+# Define the dataset name
+DATASET_NAME="self_made_dataset"
+>>>>>>> 197296f30ca7b0c49c8308bf3194348f11ab6a30
+
+# Navigate to the MATLAB script directory (assuming MATLAB can be called from command line)
+cd src/main/java/vslam
+
+# Run the MATLAB script with the dataset path
+matlab -batch "vslam_implementation('${DATASET_NAME}')"
+
+echo "VSLAM processing complete"
+

run_vslam_rgbd.sh

+#!/bin/bash
+
+# Define the script that runs the VSLAM implementation
+MATLAB_SCRIPT="/src/main/java/vslam/vslam_implementation_rgbd.m"
+
+# Check if the correct number of arguments was provided
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 {tum_rgbd_dataset|imperial_college_london}"
+    exit 1
+fi
+
+# Determine which dataset to use based on the argument provided
+if [ "$1" = "tum_rgbd_dataset" ]; then
+    DATASET_NAME="tum_rgbd_dataset"
+elif [ "$1" = "imperial_college_london" ]; then
+    DATASET_NAME="imperial_college_london"
+else
+    echo "Invalid dataset name. Choose either 'tum_rgbd_dataset' or 'imperial_college_london'"
+    exit 2
+fi
+
+# Navigate to the MATLAB script directory (assuming MATLAB can be called from command line)
+cd src/main/java/vslam
+
+# Run the MATLAB script with the dataset path
+matlab -batch "vslam_implementation_rgbd('${DATASET_NAME}')"
+
+echo "VSLAM processing complete"
+

src/main/java/EntryPoint.java

-import database.DatabaseUpdater;
-import object_detection.ObjectDetector;
-import object_detection.types.ObjectSet;
-
-public class EntryPoint {
-
-    public static void main(String args[]){
-        /*
-            - start GUI server
-            - get keyframes and featurepoints
-            - get objects from keyframes
-            - start object detection
-            - finish object detection and update database
-            - ping GUI server
-            - GUI server pulls information and displays point cloud to user
-         */
-
-        System.out.println("***********************************************");
-        System.out.println("{ Running group8's project }");
-        System.out.println("***********************************************");
-
-        // #################################################
-        // 1) Starting the GUI Server
-
-        // nothing yet
-        int port = 9999;
-
-        System.out.println(" - 1) GUI Server has been started on https://localhost:"+ port + ".");
-
-        // #################################################
-        // 2) Getting keyframes and feature points
-
-        // for now, we can just set paths to the directories that hold keyframes and featurepoint CSVs
-        String feature_csv_path = "src/main/java/vslam/KeyFramePoints";
-        String keyframe_png_path = "src/main/java/vslam/KeyFrames";
-
-        System.out.println(" - 2) Keyframe and Features have been collected.");
-
-        // #################################################
-        // 3) Get objects from keyframes
-
-        // for now, we can just set paths to the directories holding bounded box information
-        String bounding_info_path = "src/main/java/vslam/BoundedInfo";
-
-        System.out.println(" - 3) Objects have been detected via YOLOv4.");
-
-        // #################################################
-        // 4) Get objects from keyframes
-
-        // start the object detection module, output will be the objects pushed to the cloud, return the completed object set
-        ObjectSet os = ObjectDetector.process(bounding_info_path, feature_csv_path);
-
-        System.out.println(" - 4) ObjectSet has been constructed.");
-
-        // #################################################
-        // 5) Update database with objectset
-
-        // perform the update of the database using the video elements
-        boolean updateRes = DatabaseUpdater.updateDB(os);
-        if(!updateRes){
-            System.err.println("ERROR: database update failed");
-        }
-
-        System.out.println(" - 5) Database has been constructed.");
-
-        // #################################################
-        // 6) Ping GUI server with completion status, so server can start displaying ObjectSet
-
-        // nothing yet
-
-        System.out.println(" - 6) GUI will start reflecting object tracking.");
-
-
-        /*
-        At this point, everything is done, but there should be the option to rerun this system with another video possibly?
-         */
-
-        System.out.println("***********************************************");
-        System.out.println("{ Project completed with success. Hope you enjoyed. }");
-        System.out.println("***********************************************");
-    }
-}