added second transform queue

doc/api.oa3

@@ -2225,6 +2225,12 @@ paths:
           required: true
           schema:
             type: string
+        - name: confirm
+          in: query
+          description: set if intial state is confirmed
+          required: false
+          schema:
+            type: boolean
       responses:
         '201':
           description: entry created

net/server/internal/api_addChannelTopic.go

@@ -9,6 +9,8 @@ import (
 
 func AddChannelTopic(w http.ResponseWriter, r *http.Request) {
 
+  confirm := r.FormValue("confirm")
+
   var subject Subject
   if err := ParseRequest(r, w, &subject); err != nil {
     ErrResponse(w, http.StatusBadRequest, err)
@@ -33,7 +35,11 @@ func AddChannelTopic(w http.ResponseWriter, r *http.Request) {
     topic.Guid = guid
     topic.DetailRevision = act.ChannelRevision + 1
     topic.TagRevision = act.ChannelRevision + 1
-    topic.Status = APP_TOPICUNCONFIRMED
+    if confirm == "true" {
+      topic.Status = APP_TOPICCONFIRMED
+    } else {
+      topic.Status = APP_TOPICUNCONFIRMED
+    }
     topic.Transform = APP_TRANSFORMCOMPLETE
     if res := tx.Save(topic).Error; res != nil {
       return res

net/server/internal/api_addChannelTopicAsset.go

@@ -3,6 +3,7 @@ package databag
 import (
   "os"
   "io"
+  "strings"
   "errors"
   "github.com/google/uuid"
   "net/http"
@@ -103,8 +104,17 @@ func AddChannelTopicAsset(w http.ResponseWriter, r *http.Request) {
       asset.ChannelID = channelSlot.Channel.ID
       asset.TopicID = topicSlot.Topic.ID
       asset.Status = APP_ASSETWAITING
-      asset.Transform = transform
       asset.TransformId = id
+      t := strings.Split(transform, ";")
+      if len(t) > 0 {
+        asset.Transform = t[0]
+      }
+      if len(t) > 1 {
+        asset.TransformQueue = t[1]
+      }
+      if len(t) > 2 {
+        asset.TransformParams = t[2]
+      }
       if res := tx.Save(asset).Error; res != nil {
         return res
       }
@@ -135,7 +145,7 @@ func AddChannelTopicAsset(w http.ResponseWriter, r *http.Request) {
   }
 
   // invoke transcoder
-  go transcode()
+  transcode()
 
   // determine affected contact list
   cards := make(map[string]store.Card)

net/server/internal/appValues.go

@@ -35,6 +35,9 @@ const APP_ASSETPROCESSING = "processing"
 const APP_ASSETERROR = "error"
 const APP_TRANSFORMCOMPLETE = "complete"
 const APP_TRANSFORMINCOMPLETE = "incomplete"
+const APP_TRANSFORMERROR = "error"
+const APP_TRANSFORMQUEUEA = "A"
+const APP_TRANSFORMQUEUEB = "B"
 
 func AppCardStatus(status string) bool {
   if status == APP_CARDPENDING {

net/server/internal/store/schema.go

@@ -245,7 +245,8 @@ type Asset struct {
   Crc               uint32
   Transform         string
   TransformId       string
-  TransformData     string
+  TransformParams   string
+  TransformQueue    string
   Created           int64           `gorm:"autoCreateTime"`
   Updated           int64           `gorm:"autoUpdateTime"`
   Account           Account

net/server/internal/transcodeUtil.go

@@ -13,73 +13,102 @@ import (
   "gorm.io/gorm"
 )
 
-var transcodeSync sync.Mutex
+var aSync sync.Mutex
+var bSync sync.Mutex
 
 func transcode() {
 
-  transcodeSync.Lock()
+  // quick transforms should use A (eg image resize)
-  defer transcodeSync.Unlock()
+  go transcodeA()
 
-  var assets []store.Asset
+  // slow transofrms should use B (eg video transcode)
-  if err := store.DB.Preload("Account").Preload("Channel.Cards").Preload("Channel.Groups.Cards").Preload("Channel.ChannelSlot").Preload("Topic.TopicSlot").Where("status = ?", APP_ASSETWAITING).Find(&assets).Error; err != nil {
+  go transcodeB()
-    ErrMsg(err)
+}
-    return
+
+func transcodeA() {
+  aSync.Lock()
+  defer aSync.Unlock()
+
+  for ;; {
+    var asset store.Asset
+    if err := store.DB.Order("created asc").Preload("Account").Preload("Channel.Cards").Preload("Channel.Groups.Cards").Preload("Channel.ChannelSlot").Preload("Topic.TopicSlot").Where("transform_queue = ? AND status = ?", APP_TRANSFORMQUEUEA, APP_ASSETWAITING).First(&asset).Error; err != nil {
+      if !errors.Is(err, gorm.ErrRecordNotFound) {
+        ErrMsg(err)
+      }
+      return
+    }
+
+    transcodeAsset(&asset)
   }
+}
+
+func transcodeB() {
+  bSync.Lock()
+  defer bSync.Unlock()
+
+  for ;; {
+    var asset store.Asset
+    if err := store.DB.Order("created asc").Preload("Account").Preload("Channel.Cards").Preload("Channel.Groups.Cards").Preload("Channel.ChannelSlot").Preload("Topic.TopicSlot").Where("transform_queue != ? AND status = ?", APP_TRANSFORMQUEUEB, APP_ASSETWAITING).First(&asset).Error; err != nil {
+      if !errors.Is(err, gorm.ErrRecordNotFound) {
+        ErrMsg(err)
+      }
+      return
+    }
+
+    transcodeAsset(&asset)
+  }
+}
+
+func transcodeAsset(asset *store.Asset) {
 
   // prepare script path
   data := getStrConfigValue(CONFIG_ASSETPATH, ".")
   script := getStrConfigValue(CONFIG_SCRIPTPATH, ".")
   re := regexp.MustCompile("^[a-zA-Z0-9_]*$")
 
-  for _, asset := range assets {
+  if !re.MatchString(asset.Transform) {
-
+    ErrMsg(errors.New("invalid transform"))
-    if !re.MatchString(asset.Transform) {
+    if err := UpdateAsset(asset, APP_ASSETERROR, 0, 0); err != nil {
-      ErrMsg(errors.New("invalid transformi"))
+      ErrMsg(err)
-      if err := UpdateAsset(&asset, APP_ASSETERROR, 0, 0); err != nil {
+    }
+  } else {
+    input := data + "/" + asset.TransformId
+    output := data + "/" + asset.AssetId
+    cmd := exec.Command(script + "/" + asset.Transform + ".sh", input, output, asset.TransformParams)
+    var out bytes.Buffer
+    cmd.Stdout = &out
+    if err := cmd.Run(); err != nil {
+      LogMsg(out.String())
+      ErrMsg(err)
+      if err := UpdateAsset(asset, APP_ASSETERROR, 0, 0); err != nil {
         ErrMsg(err)
       }
     } else {
-      input := data + "/" + asset.TransformId
+      LogMsg(out.String())
-      output := data + "/" + asset.AssetId
+      crc, size, err := ScanAsset(output)
-      cmd := exec.Command(script + "/" + asset.Transform + ".sh", input, output)
+      if err != nil {
-      var out bytes.Buffer
-      cmd.Stdout = &out
-      if err := cmd.Run(); err != nil {
-        LogMsg(out.String())
         ErrMsg(err)
-        if err := UpdateAsset(&asset, APP_ASSETERROR, 0, 0); err != nil {
+        if err := UpdateAsset(asset, APP_ASSETERROR, 0, 0); err != nil {
-          ErrMsg(err)
-        }
-      } else {
-        LogMsg(out.String())
-        crc, size, err := ScanAsset(output)
-        if err != nil {
-          ErrMsg(err)
-          if err := UpdateAsset(&asset, APP_ASSETERROR, 0, 0); err != nil {
-            ErrMsg(err)
-          }
-        } else if err := UpdateAsset(&asset, APP_ASSETREADY, crc, size); err != nil {
           ErrMsg(err)
         }
+      } else if err := UpdateAsset(asset, APP_ASSETREADY, crc, size); err != nil {
+        ErrMsg(err)
       }
     }
   }
 }
 
-func isComplete(status string, asset *store.Asset) (complete bool, err error) {
+func isComplete(id uint) (complete bool, err error) {
-  if status == APP_ASSETREADY {
+  var assets []store.Asset
-    var assets []store.Asset
+  if err = store.DB.Where("topic_id = ?", id).Find(&assets).Error; err != nil {
-    if err = store.DB.Where("topic_id = ?", asset.Topic.ID).Find(&assets).Error; err != nil {
-      return
-    }
-    for _, a := range asset.Topic.Assets {
-      if a.ID != asset.ID && asset.Status != APP_ASSETREADY {
-        return
-      }
-    }
-    complete = true
     return
   }
+  for _, asset := range assets {
+    if id != asset.ID && asset.Status != APP_ASSETREADY {
+      return
+    }
+  }
+  complete = true
   return
 }
 
@@ -93,14 +122,20 @@ func UpdateAsset(asset *store.Asset, status string, crc uint32, size int64) (err
     if res := tx.Save(asset).Error; res != nil {
       return res
     }
-    complete, ret := isComplete(status, asset)
+    if status == APP_ASSETERROR {
-    if ret != nil {
+      if res := tx.Model(&asset.Topic).Update("transform", APP_TRANSFORMERROR).Error; res != nil {
-      return ret
-    }
-    if complete {
-      if res := tx.Model(&asset.Topic).Update("transform", APP_TRANSFORMCOMPLETE).Error; res != nil {
         return res
       }
+    } else if status == APP_ASSETREADY {
+      complete, ret := isComplete(asset.ID)
+      if ret != nil {
+        return ret
+      }
+      if complete {
+        if res := tx.Model(&asset.Topic).Update("transform", APP_TRANSFORMCOMPLETE).Error; res != nil {
+          return res
+        }
+      }
     }
     if res := tx.Model(&asset.Topic).Update("detail_revision", act.ChannelRevision + 1).Error; res != nil {
       return res

net/server/internal/ucTopicShare_test.go

@@ -92,7 +92,7 @@ func TestTopicShare(t *testing.T) {
   // add asset to topic
   assets := &[]Asset{}
   params["topicId"] = topic.Id
-  transforms, err := json.Marshal([]string{ "P01", "P02", "P03" })
+  transforms, err := json.Marshal([]string{ "P01;A;1234", "P02", "P03" })
   assert.NoError(t, err)
   assert.NoError(t, ApiTestUpload(AddChannelTopicAsset, "POST", "/content/channels/{channelId}/topics/{topicId}/assets?transforms=" + url.QueryEscape(string(transforms)),
     &params, img, APP_TOKENCONTACT, set.C.A.Token, assets, nil))