feat: add jmuxer-stream webcam type, supporting raw h264 (#1342)

Co-authored-by: Stefan Dej <meteyou@gmail.com>

package-lock.json

@@ -27,6 +27,7 @@
                 "echarts": "^5.2.2",
                 "echarts-gl": "^2.0.8",
                 "hls.js": "^1.3.3",
+                "jmuxer": "^2.0.5",
                 "js-sha256": "^0.9.0",
                 "lodash.kebabcase": "^4.1.1",
                 "lodash.throttle": "^4.1.1",
@@ -54,6 +55,7 @@
                 "@intlify/vite-plugin-vue-i18n": "^2.5.0",
                 "@mdi/js": "^7.0.0",
                 "@types/file-saver": "^2.0.5",
+                "@types/jmuxer": "^2.0.3",
                 "@types/lodash.kebabcase": "^4.1.6",
                 "@types/lodash.throttle": "^4.1.6",
                 "@types/semver": "^7.3.8",
@@ -2800,6 +2802,15 @@
             "integrity": "sha512-zv9kNf3keYegP5oThGLaPk8E081DFDuwfqjtiTzm6PoxChdJ1raSuADf2YGCVIyrSynLrgc8JWv296s7Q7pQSQ==",
             "dev": true
         },
+        "node_modules/@types/jmuxer": {
+            "version": "2.0.3",
+            "resolved": "https://registry.npmjs.org/@types/jmuxer/-/jmuxer-2.0.3.tgz",
+            "integrity": "sha512-jRwZXuPbNRG8wTfJT7gjpZm72EaMzvN3oG7zbFWdW8+OHJmxAhLyqycTd/0GtZ/P8OjKXa1cbyKxQg6coXlERg==",
+            "dev": true,
+            "dependencies": {
+                "@types/node": "*"
+            }
+        },
         "node_modules/@types/json-schema": {
             "version": "7.0.11",
             "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz",
@@ -6778,6 +6789,11 @@
                 "node": ">=8"
             }
         },
+        "node_modules/jmuxer": {
+            "version": "2.0.5",
+            "resolved": "https://registry.npmjs.org/jmuxer/-/jmuxer-2.0.5.tgz",
+            "integrity": "sha512-4qjXl8JS138WyCZZoOYwXq/qVrHcE0UIpt2lMtGyq2wuBSPMNSzP1K2CEWSrwAMgjZ9jD7Btc0SxMkiLIhoHsg=="
+        },
         "node_modules/joi": {
             "version": "17.6.2",
             "resolved": "https://registry.npmjs.org/joi/-/joi-17.6.2.tgz",
@@ -12032,6 +12048,15 @@
             "integrity": "sha512-zv9kNf3keYegP5oThGLaPk8E081DFDuwfqjtiTzm6PoxChdJ1raSuADf2YGCVIyrSynLrgc8JWv296s7Q7pQSQ==",
             "dev": true
         },
+        "@types/jmuxer": {
+            "version": "2.0.3",
+            "resolved": "https://registry.npmjs.org/@types/jmuxer/-/jmuxer-2.0.3.tgz",
+            "integrity": "sha512-jRwZXuPbNRG8wTfJT7gjpZm72EaMzvN3oG7zbFWdW8+OHJmxAhLyqycTd/0GtZ/P8OjKXa1cbyKxQg6coXlERg==",
+            "dev": true,
+            "requires": {
+                "@types/node": "*"
+            }
+        },
         "@types/json-schema": {
             "version": "7.0.11",
             "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.11.tgz",
@@ -14837,6 +14862,11 @@
                 }
             }
         },
+        "jmuxer": {
+            "version": "2.0.5",
+            "resolved": "https://registry.npmjs.org/jmuxer/-/jmuxer-2.0.5.tgz",
+            "integrity": "sha512-4qjXl8JS138WyCZZoOYwXq/qVrHcE0UIpt2lMtGyq2wuBSPMNSzP1K2CEWSrwAMgjZ9jD7Btc0SxMkiLIhoHsg=="
+        },
         "joi": {
             "version": "17.6.2",
             "resolved": "https://registry.npmjs.org/joi/-/joi-17.6.2.tgz",

package.json

@@ -42,6 +42,7 @@
         "echarts": "^5.2.2",
         "echarts-gl": "^2.0.8",
         "hls.js": "^1.3.3",
+        "jmuxer": "^2.0.5",
         "js-sha256": "^0.9.0",
         "lodash.kebabcase": "^4.1.1",
         "lodash.throttle": "^4.1.1",
@@ -69,6 +70,7 @@
         "@intlify/vite-plugin-vue-i18n": "^2.5.0",
         "@mdi/js": "^7.0.0",
         "@types/file-saver": "^2.0.5",
+        "@types/jmuxer": "^2.0.3",
         "@types/lodash.kebabcase": "^4.1.6",
         "@types/lodash.throttle": "^4.1.6",
         "@types/semver": "^7.3.8",

src/components/panels/WebcamPanel.vue

@@ -60,6 +60,9 @@
                     <template v-else-if="currentCam.service === 'hlsstream'">
                         <webcam-hlsstreamer :cam-settings="currentCam" />
                     </template>
+                    <template v-else-if="currentCam.service === 'jmuxer-stream'">
+                        <webcam-jmuxer-stream :cam-settings="currentCam" />
+                    </template>
                     <template v-else-if="currentCam.service === 'webrtc-camerastreamer'">
                         <webcam-webrtc-camerastreamer :cam-settings="currentCam" />
                     </template>
@@ -81,6 +84,7 @@ import MjpegstreamerAdaptive from '@/components/webcams/MjpegstreamerAdaptive.vu
 import Hlsstreamer from '@/components/webcams/Hlsstreamer.vue'
 import Ipstreamer from '@/components/webcams/Ipstreamer.vue'
 import Uv4lMjpeg from '@/components/webcams/Uv4lMjpeg.vue'
+import JMuxerStream from '@/components/webcams/JMuxerStream.vue'
 import WebrtcCameraStreamer from '@/components/webcams/WebrtcCameraStreamer.vue'
 import WebcamGrid from '@/components/webcams/WebcamGrid.vue'
 import Component from 'vue-class-component'
@@ -99,6 +103,7 @@ import WebcamMixin from '@/components/mixins/webcam'
         'webcam-ipstreamer': Ipstreamer,
         'webcam-hlsstreamer': Hlsstreamer,
         'webcam-uv4l-mjpeg': Uv4lMjpeg,
+        'webcam-jmuxer-stream': JMuxerStream,
         'webcam-webrtc-camerastreamer': WebrtcCameraStreamer,
         'webcam-grid': WebcamGrid,
     },

src/components/settings/SettingsWebcamsTab.vue

@@ -131,7 +131,7 @@
                                         attach></v-select>
                                 </v-col>
                             </v-row>
-                            <v-row v-if="form.service === 'mjpegstreamer-adaptive'">
+                            <v-row v-if="form.service === 'mjpegstreamer-adaptive' || form.service === 'jmuxer-stream'">
                                 <v-col class="py-2 col-6">
                                     <v-text-field
                                         v-model="form.targetFps"
@@ -190,6 +190,9 @@
                             <template v-else-if="form.service === 'hlsstream'">
                                 <webcam-hlsstreamer :cam-settings="form" />
                             </template>
+                            <template v-else-if="form.service === 'jmuxer-stream'">
+                                <webcam-jmuxer-stream :cam-settings="form" />
+                            </template>
                             <template v-else-if="form.service === 'webrtc-camerastreamer'">
                                 <webcam-webrtc-camerastreamer :cam-settings="form" />
                             </template>
@@ -228,6 +231,7 @@ import MjpegstreamerAdaptive from '@/components/webcams/MjpegstreamerAdaptive.vu
 import Uv4lMjpeg from '@/components/webcams/Uv4lMjpeg.vue'
 import WebrtcCameraStreamer from '@/components/webcams/WebrtcCameraStreamer.vue'
 import Ipstreamer from '@/components/webcams/Ipstreamer.vue'
+import JMuxerStream from '@/components/webcams/JMuxerStream.vue'
 import { mdiMenuDown, mdiDelete, mdiPencil, mdiWebcam } from '@mdi/js'
 import WebcamMixin from '@/components/mixins/webcam'
 import { FileStateFile } from '@/store/files/types'
@@ -257,6 +261,7 @@ interface webcamForm {
         'webcam-ipstreamer': Ipstreamer,
         'webcam-webrtc-camerastreamer': WebrtcCameraStreamer,
         'webcam-hlsstreamer': Hlsstreamer,
+        'webcam-jmuxer-stream': JMuxerStream,
     },
 })
 export default class SettingsWebcamsTab extends Mixins(BaseMixin, WebcamMixin) {
@@ -322,6 +327,7 @@ export default class SettingsWebcamsTab extends Mixins(BaseMixin, WebcamMixin) {
             { value: 'ipstream', text: this.$t('Settings.WebcamsTab.Ipstream') },
             { value: 'webrtc-camerastreamer', text: this.$t('Settings.WebcamsTab.WebrtcCameraStreamer') },
             { value: 'hlsstream', text: this.$t('Settings.WebcamsTab.Hlsstream') },
+            { value: 'jmuxer-stream', text: this.$t('Settings.WebcamsTab.JMuxerStream') },
         ]
     }
 

src/components/webcams/JMuxerStream.vue

@@ -0,0 +1,114 @@
+<template>
+    <div>
+        <video ref="video" autoplay :style="webcamStyle" class="webcamImage" />
+        <v-row v-if="status !== 'connected'">
+            <v-col class="_webcam_jmuxer_output text-center d-flex flex-column justify-center align-center">
+                <v-progress-circular v-if="status === 'connecting'" indeterminate color="primary" class="mb-3" />
+                <span class="mt-3">{{ status }}</span>
+            </v-col>
+        </v-row>
+    </div>
+</template>
+
+<script lang="ts">
+import JMuxer from 'jmuxer'
+import { Component, Mixins, Prop, Watch } from 'vue-property-decorator'
+import BaseMixin from '@/components/mixins/base'
+
+@Component
+export default class JMuxerStreamer extends Mixins(BaseMixin) {
+    private jmuxer: JMuxer | null = null
+    private status: string = 'connecting'
+
+    @Prop({ required: true })
+    camSettings: any
+
+    @Prop()
+    printerUrl: string | undefined
+
+    declare $refs: {
+        video: HTMLVideoElement
+    }
+
+    get url() {
+        return this.camSettings.urlStream || ''
+    }
+
+    get webcamStyle() {
+        let transforms = ''
+        if ('flipX' in this.camSettings && this.camSettings.flipX) transforms += ' scaleX(-1)'
+        if ('flipX' in this.camSettings && this.camSettings.flipY) transforms += ' scaleY(-1)'
+        if (transforms.trimStart().length) return { transform: transforms.trimStart() }
+
+        return ''
+    }
+
+    mounted() {
+        this.play()
+    }
+
+    play() {
+        this.status = 'connecting'
+        this.jmuxer?.destroy()
+
+        // Only websocket streams supported
+        if (!this.url.startsWith('ws://') && !this.url.startsWith('wss://')) {
+            console.error('jmuxer error: only websocket streams supported (ws://.. or wss://..)')
+            this.status = 'error'
+            return
+        }
+
+        const video = this.$refs.video
+        const targetFps = this.camSettings.targetFps || 10
+
+        this.jmuxer = new JMuxer({
+            node: video,
+            mode: 'video',
+            flushingTime: 0,
+            fps: targetFps,
+            // debug: true,
+            onReady: (data: any) => {
+                this.status = 'connected'
+                console.log('jmuxer ready:', data)
+            },
+            onError: (data: any) => {
+                this.status = 'error'
+                console.log('jmuxer error:', data)
+            },
+        })
+
+        const ws = new WebSocket(this.url)
+        ws.binaryType = 'arraybuffer'
+        ws.addEventListener('message', (event) => {
+            this.jmuxer?.feed({
+                video: new Uint8Array(event.data),
+            })
+        })
+
+        ws.addEventListener('error', (event) => {
+            this.status = 'error'
+            console.log('jmuxer ws error:', event)
+        })
+    }
+
+    beforeUnmount() {
+        this.jmuxer?.destroy()
+    }
+
+    @Watch('camSettings', { deep: true })
+    onCamSettingsChanged() {
+        // restart stream, when camSettings change
+        this.play()
+    }
+}
+</script>
+
+<style scoped>
+.webcamImage {
+    width: 100%;
+}
+
+._webcam_jmuxer_output {
+    aspect-ratio: calc(3 / 2);
+}
+</style>

src/components/webcams/WebcamGrid.vue

@@ -22,6 +22,9 @@
                 <template v-else-if="webcam.service === 'webrtc-camerastreamer'">
                     <webcam-webrtc-camerastreamer :cam-settings="webcam" />
                 </template>
+                <template v-else-if="webcam.service === 'jmuxer-stream'">
+                    <webcam-jmuxer-stream :cam-settings="webcam" />
+                </template>
                 <template v-else>
                     <p class="text-center py-3 font-italic">{{ $t('Panels.WebcamPanel.UnknownWebcamService') }}</p>
                 </template>
@@ -38,6 +41,7 @@ import MjpegstreamerAdaptive from '@/components/webcams/MjpegstreamerAdaptive.vu
 import Uv4lMjpeg from '@/components/webcams/Uv4lMjpeg.vue'
 import Ipstreamer from '@/components/webcams/Ipstreamer.vue'
 import Hlsstreamer from '@/components/webcams/Hlsstreamer.vue'
+import JMuxerStream from '@/components/webcams/JMuxerStream.vue'
 import WebrtcCameraStreamer from '@/components/webcams/WebrtcCameraStreamer.vue'
 import { GuiWebcamStateWebcam } from '@/store/gui/webcams/types'
 
@@ -48,6 +52,7 @@ import { GuiWebcamStateWebcam } from '@/store/gui/webcams/types'
         'webcam-uv4l-mjpeg': Uv4lMjpeg,
         'webcam-ipstreamer': Ipstreamer,
         'webcam-hlsstreamer': Hlsstreamer,
+        'webcam-jmuxer-stream': JMuxerStream,
         'webcam-webrtc-camerastreamer': WebrtcCameraStreamer,
     },
 })

src/locales/da.json

@@ -1055,6 +1055,7 @@
             "Hlsstream": "HLS Stream",
             "Mjpegstreamer": "MJPEG-Streamer",
             "MjpegstreamerAdaptive": "Adaptiv MJPEG-Streamer (eksperimental)",
+            "JMuxerStream": "Rå h264 stream (jmuxer)",
             "Name": "Navn",
             "NameAlreadyExists": "Navnet bruges allerede",
             "Required": "Krævet",

src/locales/de.json

@@ -1058,6 +1058,7 @@
             "Ipstream": "IP Kamera",
             "Mjpegstreamer": "MJPEG-Streamer",
             "MjpegstreamerAdaptive": "Adaptive MJPEG-Streamer (experimental)",
+            "JMuxerStream": "Roher h264 stream (jmuxer)",
             "Name": "Name",
             "NameAlreadyExists": "Name existiert bereits",
             "Required": "benötigt",

src/locales/en.json

@@ -1065,6 +1065,7 @@
             "Hlsstream": "HLS Stream",
             "Mjpegstreamer": "MJPEG-Streamer",
             "MjpegstreamerAdaptive": "Adaptive MJPEG-Streamer (experimental)",
+            "JMuxerStream": "Raw h264 stream (jmuxer)",
             "Name": "Name",
             "NameAlreadyExists": "Name already exists",
             "Required": "required",