playback of e2e assets in mobile app

app/mobile/ios/Databag.xcodeproj/project.pbxproj

@@ -756,7 +756,7 @@
 			buildSettings = {
 				ALWAYS_SEARCH_USER_PATHS = NO;
 				CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES;
-				CLANG_CXX_LANGUAGE_STANDARD = "c++14";
+				CLANG_CXX_LANGUAGE_STANDARD = "c++17";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_ENABLE_MODULES = YES;
 				CLANG_ENABLE_OBJC_ARC = YES;
@@ -828,7 +828,7 @@
 			buildSettings = {
 				ALWAYS_SEARCH_USER_PATHS = NO;
 				CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES;
-				CLANG_CXX_LANGUAGE_STANDARD = "c++14";
+				CLANG_CXX_LANGUAGE_STANDARD = "c++17";
 				CLANG_CXX_LIBRARY = "libc++";
 				CLANG_ENABLE_MODULES = YES;
 				CLANG_ENABLE_OBJC_ARC = YES;

app/mobile/src/context/useConversationContext.hook.js

@@ -146,7 +146,7 @@ export function useConversationContext() {
       }
       else {
         topics.current.delete(entry.id);
-        clearTopicItem(entry.id);
+        clearTopicItem(cardId, channelId, entry.id);
       }
     }
   }

app/mobile/src/context/useUploadContext.hook.js

@@ -63,14 +63,12 @@ export function useUploadContext() {
 
   const actions = {
     addTopic: (node, token, channelId, topicId, files, success, failure, cardId) => {
-      const url = cardId ?
-          `https://${node}/content/channels/${channelId}/topics/${topicId}/assets?contact=${token}` : 
-          `https://${node}/content/channels/${channelId}/topics/${topicId}/assets?agent=${token}`;
       const key = cardId ? `${cardId}:${channelId}` : `:${channelId}`; 
       const controller = new AbortController();
       const entry = {
         index: index.current,
-        url: url,
+        baseUrl: cardId ? `https://${node}/content/channels/${channelId}/topics/${topicId}/` : `https://${node}/content/channels/${channelId}/topics/${topicId}/`,
+        urlParams: cardId ? `?contact=${token}` : `?agent=${token}`,
         files,
         assets: [],
         current: null,
@@ -187,7 +185,7 @@ async function upload(entry, update, complete) {
           formData.append("asset", {uri: 'file://' + file.data, name: 'asset', type: 'application/octent-stream'});
         }
         let transform = encodeURIComponent(JSON.stringify(["ithumb;photo", "ilg;photo"]));
-        let asset = await axios.post(`${entry.url}&transforms=${transform}`, formData, {
+        let asset = await axios.post(`${entry.baseUrl}assets${entry.urlParams}&transforms=${transform}`, formData, {
           headers: { 'Content-Type': 'multipart/form-data' },
           signal: entry.cancel.signal,
           onUploadProgress: (ev) => {
@@ -213,7 +211,7 @@ async function upload(entry, update, complete) {
         }
         let thumb = 'vthumb;video;' + file.position;
         let transform = encodeURIComponent(JSON.stringify(["vlq;video", "vhd;video", thumb]));
-        let asset = await axios.post(`${entry.url}&transforms=${transform}`, formData, {
+        let asset = await axios.post(`${entry.baseUrl}assets${entry.urlParams}&transforms=${transform}`, formData, {
           headers: { 'Content-Type': 'multipart/form-data' },
           signal: entry.cancel.signal,
           onUploadProgress: (ev) => {
@@ -239,7 +237,7 @@ async function upload(entry, update, complete) {
           formData.append("asset", {uri: 'file://' + file.data, name: 'asset', type: 'application/octent-stream'});
         }
         let transform = encodeURIComponent(JSON.stringify(["acopy;audio"]));
-        let asset = await axios.post(`${entry.url}&transforms=${transform}`, formData, {
+        let asset = await axios.post(`${entry.baseUrl}assets${entry.urlParams}&transforms=${transform}`, formData, {
           headers: { 'Content-Type': 'multipart/form-data' },
           signal: entry.cancel.signal,
           onUploadProgress: (ev) => {

app/mobile/src/session/conversation/addTopic/useAddTopic.hook.js

@@ -205,24 +205,16 @@ export function useAddTopic(contentKey) {
          
           const assemble = (assets) => {
             if (!state.locked) {
-              if (assets?.length) {
               return {
-                  assets,
+                assets: assets?.length ? assets : null,
                 text: state.message,
                 textColor: state.colorSet ? state.color : null,
                 textSize: state.sizeSet ? state.size : null,
               }
             }
-              else {
-                return {
-                  text: state.message,
-                  textColor: state.colorSet ? state.color : null,
-                  textSize: state.sizeSet ? state.size : null,
-                }
-              }
-            }
             else {
               const message = {
+                assets: assets?.length ? assets : null,
                 text: state.message,
                 textColor: state.textColorSet ? state.textColor : null,
                 textSize: state.textSizeSet ? state.textSize : null,

app/mobile/src/session/conversation/topicItem/TopicItem.jsx

@@ -109,34 +109,17 @@ export function TopicItem({ item, focused, focus, hosting, remove, update, block
     );
   }
 
-
-  const renderAsset = (asset) => {
-    return (
-      <View style={styles.frame}>
-        { asset.item.image && (
-          <ImageAsset topicId={item.topicId} asset={asset.item.image} dismiss={actions.hideCarousel} />
-        )}
-        { asset.item.video && (
-          <VideoAsset topicId={item.topicId} asset={asset.item.video} dismiss={actions.hideCarousel} />
-        )}
-        { asset.item.audio && (
-          <AudioAsset topicId={item.topicId} asset={asset.item.audio} dismiss={actions.hideCarousel} />
-        )}
-      </View>
-    )
-  }
-
   const renderThumb = (thumb) => {
     return (
       <View>
-        { thumb.item.image && (
+        { thumb.item.type === 'image' && (
-          <ImageThumb topicId={item.topicId} asset={thumb.item.image} onAssetView={() => actions.showCarousel(thumb.index)} />
+          <ImageThumb url={thumb.item.thumb} onAssetView={() => actions.showCarousel(thumb.index)} />
         )}
-        { thumb.item.video && (
+        { thumb.item.type === 'video' && (
-          <VideoThumb topicId={item.topicId} asset={thumb.item.video} onAssetView={() => actions.showCarousel(thumb.index)} />
+          <VideoThumb url={thumb.item.thumb} onAssetView={() => actions.showCarousel(thumb.index)} />
         )}
-        { thumb.item.audio && (
+        { thumb.item.type === 'audio' && (
-          <AudioThumb topicId={item.topicId} asset={thumb.item.audio} onAssetView={() => actions.showCarousel(thumb.index)} />
+          <AudioThumb labe={thumb.item.label} onAssetView={() => actions.showCarousel(thumb.index)} />
         )}
       </View>
     );
@@ -236,16 +219,17 @@ export function TopicItem({ item, focused, focus, hosting, remove, update, block
             data={state.assets}
             defaultIndex={state.carouselIndex}
             scrollAnimationDuration={1000}
+            onSnapToItem={(index) => console.log('current index:', index)}
             renderItem={({ index }) => (
               <View style={styles.frame}>
-                { state.assets[index].image && (
+                { state.assets[index].type === 'image' && (
-                  <ImageAsset topicId={item.topicId} asset={state.assets[index].image} dismiss={actions.hideCarousel} />
+                  <ImageAsset asset={state.assets[index]} dismiss={actions.hideCarousel} />
                 )}
-                { state.assets[index].video && (
+                { state.assets[index].type === 'video' && (
-                  <VideoAsset topicId={item.topicId} asset={state.assets[index].video} dismiss={actions.hideCarousel} />
+                  <VideoAsset asset={state.assets[index]} dismiss={actions.hideCarousel} />
                 )}
-                { state.assets[index].audio && (
+                { state.assets[index].type === 'audio' && (
-                  <AudioAsset topicId={item.topicId} asset={state.assets[index].audio} dismiss={actions.hideCarousel} />
+                  <AudioAsset asset={state.assets[index]} dismiss={actions.hideCarousel} />
                 )}
               </View>
             )} />

app/mobile/src/session/conversation/topicItem/audioAsset/AudioAsset.jsx

@@ -1,5 +1,5 @@
 import { Image, View, Text, TouchableOpacity } from 'react-native';
-import { useRef } from 'react';
+import { useEffect, useRef } from 'react';
 import Colors from 'constants/Colors';
 import Video from 'react-native-video';
 import { useAudioAsset } from './useAudioAsset.hook';
@@ -8,9 +8,9 @@ import Icons from 'react-native-vector-icons/MaterialCommunityIcons';
 import audio from 'images/audio.png';
 import { useKeepAwake } from '@sayem314/react-native-keep-awake';
 
-export function AudioAsset({ topicId, asset, dismiss }) {
+export function AudioAsset({ asset, dismiss }) {
 
-  const { state, actions } = useAudioAsset(topicId, asset);
+  const { state, actions } = useAudioAsset(asset);
 
   const player = useRef(null);
 

app/mobile/src/session/conversation/topicItem/audioAsset/useAudioAsset.hook.js

@@ -3,7 +3,7 @@ import { ConversationContext } from 'context/ConversationContext';
 import { Image } from 'react-native';
 import { useWindowDimensions } from 'react-native';
 
-export function useAudioAsset(topicId, asset) {
+export function useAudioAsset(asset) {
 
   const [state, setState] = useState({
     width: 1,
@@ -38,9 +38,13 @@ export function useAudioAsset(topicId, asset) {
   }, [dimensions]);
 
   useEffect(() => {
-    const url = conversation.actions.getTopicAssetUrl(topicId, asset.full); 
+    if (asset.encrypted) {
-    updateState({ url });
+      updateState({ url: asset.decrypted, failed: asset.error });
-  }, [topicId, conversation, asset]);
+    }
+    else {
+      updateState({ url: asset.full });
+    }
+  }, [asset]);
 
   const actions = {
     play: () => {

app/mobile/src/session/conversation/topicItem/audioThumb/AudioThumb.jsx

@@ -4,14 +4,14 @@ import { styles } from './AudioThumb.styled';
 import Colors from 'constants/Colors';
 import audio from 'images/audio.png';
 
-export function AudioThumb({ topicId, asset, onAssetView }) {
+export function AudioThumb({ label, onAssetView }) {
 
   return (
     <TouchableOpacity activeOpacity={1} onPress={onAssetView}>
       <Image source={audio} style={{ borderRadius: 4, width: 92, height: 92, marginRight: 16, backgroundColor: Colors.lightgrey }} resizeMode={'cover'} />
-      { asset.label && (
+      { label && (
         <View style={styles.overlay}>
-          <Text style={styles.label}>{ asset.label }</Text>
+          <Text style={styles.label}>{ label }</Text>
         </View>
       )}
     </TouchableOpacity>

app/mobile/src/session/conversation/topicItem/imageAsset/ImageAsset.jsx

@@ -4,8 +4,8 @@ import { styles } from './ImageAsset.styled';
 import Colors from 'constants/Colors';
 import Ionicons from 'react-native-vector-icons/AntDesign';
 
-export function ImageAsset({ topicId, asset, dismiss }) {
+export function ImageAsset({ asset, dismiss }) {
-  const { state, actions } = useImageAsset(topicId, asset);
+  const { state, actions } = useImageAsset(asset);
 
   return (
     <TouchableOpacity style={styles.container} activeOpacity={1} onPress={actions.showControls}>
@@ -33,4 +33,3 @@ export function ImageAsset({ topicId, asset, dismiss }) {
     </TouchableOpacity>
   );
 }
-  

app/mobile/src/session/conversation/topicItem/imageAsset/useImageAsset.hook.js

@@ -3,7 +3,7 @@ import { ConversationContext } from 'context/ConversationContext';
 import { Image } from 'react-native';
 import { useWindowDimensions } from 'react-native';
 
-export function useImageAsset(topicId, asset) {
+export function useImageAsset(asset) {
 
   const [state, setState] = useState({
     frameWidth: 1,
@@ -49,9 +49,13 @@ export function useImageAsset(topicId, asset) {
   }, [dimensions]);
 
   useEffect(() => {
-    const url = conversation.actions.getTopicAssetUrl(topicId, asset.full); 
+    if (asset.encrypted) {
-    updateState({ url });
+      updateState({ url: asset.decrypted, failed: asset.error });
-  }, [topicId, conversation, asset]);
+    }
+    else {
+      updateState({ url: asset.full });
+    }
+  }, [asset]);
 
   const actions = {
     loaded: (e) => {

app/mobile/src/session/conversation/topicItem/imageThumb/ImageThumb.jsx

@@ -4,12 +4,12 @@ import { useImageThumb } from './useImageThumb.hook';
 import { styles } from './ImageThumb.styled';
 import Colors from 'constants/Colors';
 
-export function ImageThumb({ topicId, asset, onAssetView }) {
+export function ImageThumb({ url, onAssetView }) {
-  const { state, actions } = useImageThumb(topicId, asset);
+  const { state, actions } = useImageThumb();
 
   return (
     <TouchableOpacity activeOpacity={1} onPress={onAssetView}>
-      <Image source={{ uri: state.url }} style={{ opacity: state.loaded ? 1 : 0, borderRadius: 4, width: 92 * state.ratio, height: 92, marginRight: 16 }}
+      <Image source={{ uri: url }} style={{ opacity: state.loaded ? 1 : 0, borderRadius: 4, width: 92 * state.ratio, height: 92, marginRight: 16 }}
           onLoad={actions.loaded} resizeMode={'cover'} />
     </TouchableOpacity>
   );

app/mobile/src/session/conversation/topicItem/imageThumb/useImageThumb.hook.js

@@ -2,7 +2,7 @@ import { useState, useRef, useEffect, useContext } from 'react';
 import { ConversationContext } from 'context/ConversationContext';
 import { Image } from 'react-native';
 
-export function useImageThumb(topicId, asset) {
+export function useImageThumb() {
 
   const [state, setState] = useState({
     loaded: false,
@@ -16,11 +16,6 @@ export function useImageThumb(topicId, asset) {
     setState((s) => ({ ...s, ...value }));
   }
 
-  useEffect(() => {
-    const url = conversation.actions.getTopicAssetUrl(topicId, asset.thumb); 
-    updateState({ url });
-  }, [topicId, conversation, asset]);
-
   const actions = {
     loaded: (e) => {
       const { width, height } = e.nativeEvent.source;

app/mobile/src/session/conversation/topicItem/useTopicItem.hook.js

@@ -1,4 +1,4 @@
-import { useState, useEffect, useContext } from 'react';
+import { useRef, useState, useEffect, useContext } from 'react';
 import { Linking } from 'react-native';
 import { ConversationContext } from 'context/ConversationContext';
 import { CardContext } from 'context/CardContext';
@@ -8,10 +8,12 @@ import moment from 'moment';
 import { useWindowDimensions, Text } from 'react-native';
 import Colors from 'constants/Colors';
 import { getCardByGuid } from 'context/cardUtil';
-import { decryptTopicSubject } from 'context/sealUtil';
+import { decryptBlock, decryptTopicSubject } from 'context/sealUtil';
 import { sanitizeUrl } from '@braintree/sanitize-url';
 import Share from 'react-native-share';
 import RNFetchBlob from "rn-fetch-blob";
+import RNFS from 'react-native-fs';
+import { checkResponse, fetchWithTimeout } from 'api/fetchUtil';
 
 export function useTopicItem(item, hosting, remove, contentKey) {
 
@@ -42,6 +44,8 @@ export function useTopicItem(item, hosting, remove, contentKey) {
   const account = useContext(AccountContext);
   const dimensions = useWindowDimensions();
 
+  const cancel = useRef(false);
+
   const updateState = (value) => {
     setState((s) => ({ ...s, ...value }));
   }
@@ -50,6 +54,43 @@ export function useTopicItem(item, hosting, remove, contentKey) {
     updateState({ width: dimensions.width, height: dimensions.height });
   }, [dimensions]);
 
+  const setAssets = (parsed) => {
+    const assets = [];
+    if (parsed?.length) {
+      for (let i = 0; i < parsed.length; i++) {
+        const asset = parsed[i];
+        if (asset.encrypted) {
+          const encrypted = true;
+          const { type, thumb, label, parts } = asset.encrypted;
+          assets.push({ type, thumb, label, encrypted, decrypted: null, parts });
+        }
+        else {
+          const encrypted = false
+          if (asset.image) {
+            const type = 'image';
+            const thumb = conversation.actions.getTopicAssetUrl(item.topicId, asset.image.thumb);
+            const full = conversation.actions.getTopicAssetUrl(item.topicId, asset.image.full);
+            assets.push({ type, thumb, encrypted, full });
+          }
+          else if (asset.video) {
+            const type = 'video';
+            const thumb = conversation.actions.getTopicAssetUrl(item.topicId, asset.video.thumb);
+            const lq = conversation.actions.getTopicAssetUrl(item.topicId, asset.video.lq);
+            const hd = conversation.actions.getTopicAssetUrl(item.topicId, asset.video.hd);
+            assets.push({ type, thumb, encrypted, lq, hd });
+          }
+          else if (asset.audio) {
+            const type = 'audio';
+            const label = asset.audio.label;
+            const full = conversation.actions.getTopicAssetUrl(item.topicId, asset.audio.full);
+            assets.push({ type, label, encrypted, full });
+          }
+        }
+      };
+    }
+    return assets;
+  }
+
   useEffect(() => {
 
     const { topicId, revision, detail, unsealedDetail } = item;
@@ -103,7 +144,7 @@ export function useTopicItem(item, hosting, remove, contentKey) {
         parsed = JSON.parse(data);
         message = parsed?.text;
         clickable = clickableText(parsed.text);
-        assets = parsed.assets;
+        assets = setAssets(parsed.assets);
         if (parsed.textSize === 'small') {
           fontSize = 10;
         }
@@ -145,6 +186,7 @@ export function useTopicItem(item, hosting, remove, contentKey) {
       if (unsealed) {
         sealed = false;
         parsed = unsealed.message;
+        assets = setAssets(parsed.assets);
         message = parsed?.text;
         clickable = clickableText(parsed?.text);
         if (parsed?.textSize === 'small') {
@@ -231,11 +273,50 @@ export function useTopicItem(item, hosting, remove, contentKey) {
   };
 
   const actions = {
-    showCarousel: (index) => {
+    showCarousel: async (index) => {
-      updateState({ carousel: true, carouselIndex: index });
+      const assets = state.assets.map((asset) => ({ ...asset, error: false, decrypted: null }));
+      updateState({ assets, carousel: true, carouselIndex: index });
+
+      try {
+        cancel.current = false;
+        const assets = state.assets;
+        for (let i = 0; i < assets.length; i++) {
+          const cur = (i + index) % assets.length
+          const asset = assets[cur];
+          if (asset.encrypted) {
+            const ext = asset.type === 'video' ? '.mp4' : asset.type === 'audio' ? '.mp3' : '';
+            const path = RNFS.TemporaryDirectoryPath + `/${i}.asset${ext}`;
+            const exists = await RNFS.exists(path);
+            if (exists) {
+              RNFS.unlink(path);
+            }
+            for (let j = 0; j < asset.parts.length; j++) {
+              const part = asset.parts[j];
+              const url = conversation.actions.getTopicAssetUrl(item.topicId, part.partId);
+              const response = await fetchWithTimeout(url, { method: 'GET' });
+              const block = await response.text();
+              const decrypted = decryptBlock(block, part.blockIv, contentKey);
+              if (cancel.current) {
+                throw new Error("unseal assets cancelled");
+              }
+              await RNFS.appendFile(path, decrypted, 'base64');
+            };
+
+            asset.decrypted = path;
+            assets[cur] = { ...asset };
+            updateState({ assets: [ ...assets ]});
+          };
+        }
+      }
+      catch (err) {
+        console.log(err);
+        const assets = state.assets.map((asset) => ({ ...asset, error: true }));
+        updateState({ assets: [ ...assets ]});
+      }
     },
     hideCarousel: () => {
       updateState({ carousel: false });
+      cancel.current = true;
     },
     setActive: (activeId) => {
       updateState({ activeId });

app/mobile/src/session/conversation/topicItem/videoAsset/VideoAsset.jsx

@@ -6,9 +6,11 @@ import { styles } from './VideoAsset.styled';
 import Icons from 'react-native-vector-icons/MaterialCommunityIcons';
 import { useKeepAwake } from '@sayem314/react-native-keep-awake';
 
-export function VideoAsset({ topicId, asset, dismiss }) {
+import { useEffect } from 'react';
 
-  const { state, actions } = useVideoAsset(topicId, asset);
+export function VideoAsset({ asset, dismiss }) {
+
+  const { state, actions } = useVideoAsset(asset);
 
   useKeepAwake();
 
@@ -16,7 +18,7 @@ export function VideoAsset({ topicId, asset, dismiss }) {
     <View style={styles.container}>
       <TouchableOpacity activeOpacity={1} style={styles.container}  onPress={actions.showControls}>
         { state.url && (
-          <Video source={{ uri: state.url }} style={{ width: state.width, height: state.height }} resizeMode={'cover'} 
+          <Video source={{ uri: state.url, type: 'video/mp4' }} style={{ width: state.width, height: state.height }} resizeMode={'cover'} 
             onReadyForDisplay={(e) => { console.log(e) }}
             onLoad={actions.loaded} repeat={true} paused={!state.playing} resizeMode="contain" />
         )}

app/mobile/src/session/conversation/topicItem/videoAsset/useVideoAsset.hook.js

@@ -3,7 +3,7 @@ import { ConversationContext } from 'context/ConversationContext';
 import { Image } from 'react-native';
 import { useWindowDimensions } from 'react-native';
 
-export function useVideoAsset(topicId, asset) {
+export function useVideoAsset(asset) {
 
   const [state, setState] = useState({
     frameWidth: 1,
@@ -46,9 +46,13 @@ export function useVideoAsset(topicId, asset) {
   }, [dimensions]);
 
   useEffect(() => {
-    const url = conversation.actions.getTopicAssetUrl(topicId, asset.hd); 
+    if (asset.encrypted) {
-    updateState({ url });
+      updateState({ url: asset.decrypted, failed: asset.error });
-  }, [topicId, conversation, asset]);
+    }
+    else {
+      updateState({ url: asset.hd });
+    }
+  }, [asset]);
 
   const actions = {
     setResolution: (width, height) => {

app/mobile/src/session/conversation/topicItem/videoThumb/VideoThumb.jsx

@@ -5,12 +5,13 @@ import { styles } from './VideoThumb.styled';
 import Colors from 'constants/Colors';
 import AntIcons from 'react-native-vector-icons/AntDesign';
 
-export function VideoThumb({ topicId, asset, onAssetView }) {
+export function VideoThumb({ url, onAssetView }) {
-  const { state, actions } = useVideoThumb(topicId, asset);
+  const { state, actions } = useVideoThumb();
 
   return (
     <TouchableOpacity activeOpacity={1} onPress={onAssetView}>
-      <Image source={{ uri: state.url }} style={{ borderRadius: 4, width: 92 * state.ratio, height: 92, marginRight: 16 }} resizeMode={'cover'} />
+      <Image source={{ uri: url }} style={{ opacity: state.loaded ? 1 : 0, borderRadius: 4, width: 92 * state.ratio, height: 92, marginRight: 16 }}
+          onLoad={actions.loaded} resizeMode={'cover'} />
       <View style={styles.overlay}>
         <AntIcons name="caretright" size={20} color={Colors.white} />
       </View>

app/mobile/src/session/conversation/topicItem/videoThumb/useVideoThumb.hook.js

@@ -2,11 +2,10 @@ import { useState, useRef, useEffect, useContext } from 'react';
 import { ConversationContext } from 'context/ConversationContext';
 import { Image } from 'react-native';
 
-export function useVideoThumb(topicId, asset) {
+export function useVideoThumb() {
 
   const [state, setState] = useState({
     ratio: 1,
-    url: null,
   });
 
   const conversation = useContext(ConversationContext);
@@ -15,16 +14,11 @@ export function useVideoThumb(topicId, asset) {
     setState((s) => ({ ...s, ...value }));
   }
 
-  useEffect(() => {
-    const url = conversation.actions.getTopicAssetUrl(topicId, asset.thumb); 
-    if (url) {
-      Image.getSize(url, (width, height) => {
-        updateState({ url, ratio: width / height });
-      });
-    }
-  }, [topicId, conversation, asset]);
-
   const actions = {
+    loaded: (e) => {
+      const { width, height } = e.nativeEvent.source;
+      updateState({ loaded: true, ratio: width / height });
+    },
   };
 
   return { state, actions };