Merge "Use database to store ota generate history."

9 files changed, 331 insertions, 89 deletions

diff --git a/tools/otagui/ota_interface.py b/tools/otagui/ota_interface.py
index f5b5fb036..ec0490629 100644
--- a/tools/otagui/ota_interface.py
+++ b/tools/otagui/ota_interface.py
@@ -2,52 +2,170 @@ import subprocess
 import os
 import json
 import pipes
-from threading import Lock
+import threading
+from dataclasses import dataclass, asdict, field
 import logging
+import sqlite3
+import time
 
-class ProcessesManagement:
-    def __init__(self):
-        self.__container = {}
-        self.__lock = Lock()
 
-    def set(self, name, value):
-        with self.__lock:
-            self.__container[name] = value
+@dataclass
+class JobInfo:
+    """
+    A class for ota job information
+    """
+    id: str
+    target: str
+    incremental: str = ''
+    verbose: bool = False
+    partial: list[str] = field(default_factory=list)
+    output: str = ''
+    status: str = 'Running'
+    downgrade: bool = False
+    extra: str = ''
+    stdout: str = ''
+    stderr: str = ''
+    start_time: int = 0
+    finish_time: int = 0
+    isPartial: bool = False
+    isIncremental: bool = False
 
-    def get(self, name):
-        with self.__lock:
-            return self.__container[name]
+    def __post_init__(self):
+        if not self.output:
+            self.output = os.path.join('output', self.id, '.zip')
+        if not self.stdout:
+            self.stdout = os.path.join('output/stdout.'+self.id)
+        if not self.stderr:
+            self.stderr = os.path.join('output/stderr.'+self.id)
 
-    def get_keys(self):
-        with self.__lock:
-            return self.__container.keys()
+        def enforce_bool(t): return t if isinstance(t, bool) else bool(t)
+        self.verbose, self.downgrade = map(
+            enforce_bool,
+            [self.verbose, self.downgrade])
+        if self.incremental:
+            self.isIncremental = True
+        if self.partial:
+            self.isPartial = True
 
-    def get_status_by_ID(self, id=0, details=False):
-        status = {}
-        if not id in self.get_keys():
-            return '{}'
-        else:
-            status['id'] = id
-            if self.get(id).poll() == None:
-                status['status'] = 'Running'
-            elif self.get(id).poll() == 0:
-                status['status'] = 'Finished'
-                status['path'] = os.path.join('output', str(id) + '.zip')
-            else:
-                status['status'] = 'Error'
-            try:
-                if details:
-                    with open(os.path.join('output', 'stdout.' + str(id)), 'r') as fout:
-                        status['stdout'] = fout.read()
-                    with open(os.path.join('output', 'stderr.' + str(id)), 'r') as ferr:
-                        status['stderr'] = ferr.read()
-            except FileNotFoundError:
-                status['stdout'] = 'NO STD OUTPUT IS FOUND'
-                status['stderr'] = 'NO STD OUTPUT IS FOUND'
-            return status
+    def to_sql_form_dict(self):
+        sql_form_dict = asdict(self)
+        sql_form_dict['partial'] = ','.join(sql_form_dict['partial'])
+        def bool_to_int(t): return 1 if t else 0
+        sql_form_dict['verbose'], sql_form_dict['downgrade'] = map(
+            bool_to_int,
+            [sql_form_dict['verbose'], sql_form_dict['downgrade']])
+        return sql_form_dict
+
+    def to_dict_basic(self):
+        basic_info = asdict(self)
+        basic_info['target_name'] = self.target.split('/')[-1]
+        if self.isIncremental:
+            basic_info['incremental_name'] = self.incremental.split('/')[-1]
+        return basic_info
+
+    def to_dict_detail(self, target_lib, offset=0):
+        detail_info = asdict(self)
+        try:
+            with open(self.stdout, 'r') as fout:
+                detail_info['stdout'] = fout.read()
+            with open(self.stderr, 'r') as ferr:
+                detail_info['stderr'] = ferr.read()
+        except FileNotFoundError:
+            detail_info['stdout'] = 'NO STD OUTPUT IS FOUND'
+            detail_info['stderr'] = 'NO STD ERROR IS FOUND'
+        target_info = target_lib.get_build_by_path(self.target)
+        detail_info['target_name'] = target_info.file_name
+        detail_info['target_build_version'] = target_info.build_version
+        if self.incremental:
+            incremental_info = target_lib.get_build_by_path(
+                self.incremental)
+            detail_info['incremental_name'] = incremental_info.file_name
+            detail_info['incremental_build_version'] = incremental_info.build_version
+        return detail_info
+
+
+class ProcessesManagement:
+    """
+    A class manage the ota generate process
+    """
+
+    def __init__(self, path='ota_database.db'):
+        """
+        create a table if not exist
+        """
+        self.path = path
+        with sqlite3.connect(self.path) as connect:
+            cursor = connect.cursor()
+            cursor.execute("""
+                CREATE TABLE if not exists Jobs (
+                ID TEXT,
+                TargetPath TEXT,
+                IncrementalPath TEXT,
+                Verbose INTEGER,
+                Partial TEXT,
+                OutputPath TEXT,
+                Status TEXT,
+                Downgrade INTEGER,
+                OtherFlags TEXT,
+                STDOUT TEXT,
+                STDERR TEXT,
+                StartTime INTEGER,
+                FinishTime INTEGER
+            )
+            """)
+
+    def get_status_by_ID(self, id):
+        with sqlite3.connect(self.path) as connect:
+            cursor = connect.cursor()
+            logging.info(id)
+            cursor.execute("""
+            SELECT ID, TargetPath, IncrementalPath, Verbose, Partial, OutputPath, Status, Downgrade, OtherFlags, STDOUT, STDERR, StartTime, FinishTime
+            FROM Jobs WHERE ID=(?)
+            """, (id,))
+            row = cursor.fetchone()
+        status = JobInfo(*row)
+        return status
 
     def get_status(self):
-        return [self.get_status_by_ID(id=id) for id in self.get_keys()]
+        with sqlite3.connect(self.path) as connect:
+            cursor = connect.cursor()
+            cursor.execute("""
+            SELECT ID, TargetPath, IncrementalPath, Verbose, Partial, OutputPath, Status, Downgrade, OtherFlags, STDOUT, STDERR, StartTime, FinishTime
+            FROM Jobs
+            """)
+            rows = cursor.fetchall()
+        statuses = [JobInfo(*row) for row in rows]
+        return statuses
+
+    def update_status(self, id, status, finish_time):
+        with sqlite3.connect(self.path) as connect:
+            cursor = connect.cursor()
+            cursor.execute("""
+                UPDATE Jobs SET Status=(?), FinishTime=(?)
+                WHERE ID=(?)
+                """,
+                           (status, finish_time, id))
+
+    def ota_run(self, command, id):
+        # Start a subprocess and collect the output
+        stderr_pipes = pipes.Template()
+        stdout_pipes = pipes.Template()
+        ferr = stderr_pipes.open(os.path.join(
+            'output', 'stderr.'+str(id)), 'w')
+        fout = stdout_pipes.open(os.path.join(
+            'output', 'stdout.'+str(id)), 'w')
+        try:
+            proc = subprocess.Popen(
+                command, stderr=ferr, stdout=fout)
+        except FileNotFoundError:
+            logging.error('ota_from_target_files is not set properly')
+            self.update_status(id, 'Error', int(time.time()))
+            return
+        exit_code = proc.wait()
+        if exit_code == 0:
+            self.update_status(id, 'Finished', int(time.time()))
+        else:
+            self.update_status(id, 'Error', int(time.time()))
 
     def ota_generate(self, args, id=0):
         command = ['ota_from_target_files']
@@ -71,15 +189,28 @@ class ProcessesManagement:
             command.append(args['partial'])
         command.append(args['target'])
         command.append(args['output'])
-        # Start a subprocess and collect the output
-        stderr_pipes = pipes.Template()
-        stdout_pipes = pipes.Template()
-        ferr = stderr_pipes.open(os.path.join(
-            'output', 'stderr.'+str(id)), 'w')
-        fout = stdout_pipes.open(os.path.join(
-            'output', 'stdout.'+str(id)), 'w')
-        self.set(id, subprocess.Popen(
-            command, stderr=ferr, stdout=fout))
+        job_info = JobInfo(id,
+                           target=args['target'],
+                           incremental=args['incremental'] if args['isIncremental'] else '',
+                           verbose=args['verbose'],
+                           partial=args['partial'].split(
+                               ' ') if args['isPartial'] else [],
+                           output=args['output'],
+                           status='Running',
+                           extra=args['extra'],
+                           start_time=int(time.time())
+                           )
+        try:
+            thread = threading.Thread(target=self.ota_run, args=(command, id))
+            with sqlite3.connect(self.path) as connect:
+                cursor = connect.cursor()
+                cursor.execute("""
+                    INSERT INTO Jobs (ID, TargetPath, IncrementalPath, Verbose, Partial, OutputPath, Status, Downgrade, OtherFlags, STDOUT, STDERR, StartTime)
+                    VALUES (:id, :target, :incremental, :verbose, :partial, :output, :status, :downgrade, :extra, :stdout, :stderr, :start_time)
+                """, job_info.to_sql_form_dict())
+            thread.start()
+        except AssertionError:
+            raise SyntaxError
         logging.info(
             'Starting generating OTA package with id {}: \n {}'
-            .format(id, command))
\ No newline at end of file
+            .format(id, command))
diff --git a/tools/otagui/src/components/JobConfiguration.vue b/tools/otagui/src/components/JobConfiguration.vue
new file mode 100644
index 000000000..17e660637
--- /dev/null
+++ b/tools/otagui/src/components/JobConfiguration.vue
@@ -0,0 +1,47 @@
+<template>
+  <ul v-if="job">
+    <li>Start Time: {{ formDate(job.start_time) }}</li>
+    <li v-if="job.finish_time > 0">
+      Finish Time: {{ formDate(job.finish_time) }}
+    </li>
+    <li v-if="job.isIncremental">
+      Incremental source: {{ job.incremental_name }}
+    </li>
+    <li v-if="job.isIncremental && buildDetail">
+      Incremental source version: {{ job.incremental_build_version }}
+    </li>
+    <li>Target source: {{ job.target_name }}</li>
+    <li v-if="buildDetail">
+      Target source version: {{ job.target_build_version }}
+    </li>
+    <li v-if="job.isPartial">
+      Partial: {{ job.partial }}
+    </li>
+  </ul>
+</template>
+
+<script>
+import FormDate from '../services/FormDate.js'
+
+export default {
+  components: {
+    FormDate,
+  },
+  props: {
+    job: {
+      type: Object,
+      required: true,
+      default: null,
+    },
+    buildDetail: {
+      type: Boolean,
+      default: false,
+    },
+  },
+  methods: {
+    formDate(unixTime) {
+      return FormDate.formDate(unixTime)
+    },
+  },
+}
+</script>
\ No newline at end of file
diff --git a/tools/otagui/src/components/JobDisplay.vue b/tools/otagui/src/components/JobDisplay.vue
index a1f9eafdd..83a1aa812 100644
--- a/tools/otagui/src/components/JobDisplay.vue
+++ b/tools/otagui/src/components/JobDisplay.vue
@@ -1,26 +1,39 @@
 <template>
-  <router-link
-    :to="{ name: 'JobDetails', params: {id: job.id} }"
-  >
+  <router-link :to="{ name: 'JobDetails', params: { id: job.id } }">
     <div class="job-display">
       <span>Status of Job.{{ job.id }}</span>
       <h4>{{ job.status }}</h4>
+      <div v-show="active">
+        <JobConfiguration
+          :job="job"
+          :build-detail="false"
+        />
+      </div>
     </div>
   </router-link>
 </template>
 
 <script>
+import JobConfiguration from '../components/JobConfiguration.vue'
+
 export default {
+  components: {
+    JobConfiguration
+  },
   props: {
     job: {
       type: Object,
-      required: true
-    }
+      required: true,
+    },
+    active: {
+      type: Boolean,
+      default: false,
+    },
   }
 }
 </script>
 
-<style scoped>
+<style>
 .job-display {
   padding: 20px;
   width: 250px;
diff --git a/tools/otagui/src/services/FormDate.js b/tools/otagui/src/services/FormDate.js
new file mode 100644
index 000000000..930b83689
--- /dev/null
+++ b/tools/otagui/src/services/FormDate.js
@@ -0,0 +1,18 @@
+export default{
+  formDate(unixTime) {
+    let formTime = new Date(unixTime * 1000)
+    let date =
+      formTime.getFullYear() +
+      '-' +
+      (formTime.getMonth() + 1) +
+      '-' +
+      formTime.getDate()
+    let time =
+      formTime.getHours() +
+      ':' +
+      formTime.getMinutes() +
+      ':' +
+      formTime.getSeconds()
+    return date + ' ' + time
+  }
+}
\ No newline at end of file
diff --git a/tools/otagui/src/views/JobDetails.vue b/tools/otagui/src/views/JobDetails.vue
index 32eaa72e1..89abc0901 100644
--- a/tools/otagui/src/views/JobDetails.vue
+++ b/tools/otagui/src/views/JobDetails.vue
@@ -1,16 +1,26 @@
 <template>
   <div v-if="job">
     <h3>Job. {{ job.id }} {{ job.status }}</h3>
+    <JobConfiguration
+      :job="job"
+      :build-detail="true"
+    />
     <div>
       <h4>STDERR</h4>
-      <div class="stderr">
+      <div
+        ref="stderr"
+        class="stderr"
+      >
         {{ job.stderr }}
-        <p ref="stderr_bottom" />
+        <p ref="stderrBottom" />
       </div>
       <h4>STDOUT</h4>
-      <div class="stdout">
+      <div
+        ref="stdout"
+        class="stdout"
+      >
         {{ job.stdout }}
-        <p ref="stdout_bottom" />
+        <p ref="stdoutBottom" />
       </div>
     </div>
     <br>
@@ -22,9 +32,23 @@
 </template>
 
 <script>
+import { ref } from 'vue'
 import ApiService from '../services/ApiService.js'
+import JobConfiguration from '../components/JobConfiguration.vue'
+
 export default {
+  components: {
+    ApiService,
+    JobConfiguration,
+  },
   props: ['id'],
+  setup() {
+    const stderr = ref()
+    const stdout = ref()
+    const stderrBottom = ref()
+    const stdoutBottom = ref()
+    return { stderr, stdout, stderrBottom, stdoutBottom }
+  },
   data() {
     return {
       job: null,
@@ -32,7 +56,7 @@ export default {
   },
   computed: {
     download() {
-      return 'http://localhost:8000/download/' + this.job.path
+      return 'http://localhost:8000/download/' + this.job.output
     },
   },
   created() {
@@ -44,15 +68,27 @@ export default {
       try {
         let response = await ApiService.getJobById(this.id)
         this.job = response.data
-        await this.$refs.stdout_bottom.scrollIntoView({ behavior: 'smooth' })
-        await this.$refs.stderr_bottom.scrollIntoView({ behavior: 'smooth' })
+      } catch (err) {
+        console.log(err)
+      }
+      try {
+        await this.$nextTick(() => {
+          this.stderr.scrollTo({
+            top: this.stderrBottom.offsetTop,
+            behavior: 'smooth',
+          })
+          this.stdout.scrollTo({
+            top: this.stdoutBottom.offsetTop,
+            behavior: 'smooth',
+          })
+        })
       } catch (err) {
         console.log(err)
       }
       if (this.job.status == 'Running') {
         setTimeout(this.updateStatus, 1000)
       }
-    },
+    }
   },
 }
 </script>
@@ -60,7 +96,7 @@ export default {
 <style scoped>
 .stderr,
 .stdout {
-	overflow: scroll;
-	height: 200px;
+  overflow: scroll;
+  height: 200px;
 }
 </style>
\ No newline at end of file
diff --git a/tools/otagui/src/views/JobList.vue b/tools/otagui/src/views/JobList.vue
index 9f58095a1..53450a944 100644
--- a/tools/otagui/src/views/JobList.vue
+++ b/tools/otagui/src/views/JobList.vue
@@ -4,6 +4,9 @@
       v-for="job in jobs"
       :key="job.id"
       :job="job"
+      :active="overStatus.get(job.id)"
+      @mouseover="mouseOver(job.id, true)"
+      @mouseout="mouseOver(job.id, false)"
     />
     <button @click="updateStatus">
       Update
@@ -23,6 +26,7 @@ export default {
   data() {
     return {
       jobs: null,
+      overStatus: new Map()
     }
   },
   created (){
@@ -36,6 +40,9 @@ export default {
       } catch (err) {
         console.log(err);
       }
+    },
+    mouseOver(id, status) {
+      this.overStatus.set(id, status)
     }
   }
 }
diff --git a/tools/otagui/src/views/SimpleForm.vue b/tools/otagui/src/views/SimpleForm.vue
index 6ec1e7481..b6f6b045e 100644
--- a/tools/otagui/src/views/SimpleForm.vue
+++ b/tools/otagui/src/views/SimpleForm.vue
@@ -102,6 +102,7 @@ import FileSelect from '@/components/FileSelect.vue'
 import ApiService from '../services/ApiService.js'
 import UploadFile from '@/components/UploadFile.vue'
 import PartialCheckbox from '@/components/PartialCheckbox.vue'
+import FormDate from '../services/FormDate.js'
 import { uuid } from 'vue-uuid'
 
 export default {
@@ -111,6 +112,7 @@ export default {
     UploadFile,
     FileSelect,
     PartialCheckbox,
+    FormDate,
   },
   data() {
     return {
@@ -162,7 +164,8 @@ export default {
         partial: '',
         isPartial: false,
         extra: '',
-      }
+      },
+      this.partitionInclude = new Map()
     },
     async sendForm(e) {
       try {
@@ -189,20 +192,7 @@ export default {
       this.input.output += String(this.id) + '.zip'
     },
     formDate(unixTime) {
-      let formTime = new Date(unixTime * 1000)
-      let date =
-        formTime.getFullYear() +
-        '-' +
-        (formTime.getMonth() + 1) +
-        '-' +
-        formTime.getDate()
-      let time =
-        formTime.getHours() +
-        ':' +
-        formTime.getMinutes() +
-        ':' +
-        formTime.getSeconds()
-      return date + ' ' + time
+      return FormDate.formDate(unixTime)
     },
     selectTarget(path) {
       this.input.target = path
diff --git a/tools/otagui/target_lib.py b/tools/otagui/target_lib.py
index 76d7a8208..02d62abb9 100644
--- a/tools/otagui/target_lib.py
+++ b/tools/otagui/target_lib.py
@@ -68,6 +68,9 @@ class BuildInfo:
 
 
 class TargetLib:
+    """
+    A class that manages the builds in database.
+    """
     def __init__(self, path='ota_database.db'):
         """
         Create a build table if not existing
@@ -142,7 +145,7 @@ class TargetLib:
             FROM Builds""")
             return list(map(self.sql_to_buildinfo, cursor.fetchall()))
 
-    def get_builds_by_path(self, path):
+    def get_build_by_path(self, path):
         """
         Get a build in the database by its path
         Return:
@@ -153,6 +156,6 @@ class TargetLib:
             cursor = connect.cursor()
             cursor.execute("""
             SELECT FileName, Path, UploadTime, BuildID, BuildVersion, BuildFlavor, Partitions
-            WHERE Path==(?)
+            FROM Builds WHERE Path==(?)
             """, (path, ))
         return self.sql_to_buildinfo(cursor.fetchone())
diff --git a/tools/otagui/web_server.py b/tools/otagui/web_server.py
index 79c359903..bffc46f77 100644
--- a/tools/otagui/web_server.py
+++ b/tools/otagui/web_server.py
@@ -66,22 +66,19 @@ class RequestHandler(CORSSimpleHTTPHandler):
     def do_GET(self):
         if self.path.startswith('/check'):
             if self.path == '/check' or self.path == '/check/':
-                status = jobs.get_status()
+                statuses = jobs.get_status()
                 self._set_response(type='application/json')
                 self.wfile.write(
-                    json.dumps(status).encode()
+                    json.dumps([status.to_dict_basic()
+                               for status in statuses]).encode()
                 )
             else:
                 id = self.path[7:]
-                status = jobs.get_status_by_ID(id=id, details=True)
+                status = jobs.get_status_by_ID(id=id)
                 self._set_response(type='application/json')
                 self.wfile.write(
-                    json.dumps(status).encode()
+                    json.dumps(status.to_dict_detail(target_lib)).encode()
                 )
-            logging.info(
-                "GET request:\nPath:%s\nHeaders:\n%s\nBody:\n%s\n",
-                str(self.path), str(self.headers), status
-            )
             return
         elif self.path.startswith('/file'):
             if self.path == '/file' or self.path == '/file/':