Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit2d4eec1

Browse files
committed
fix: MP4Clip has not adapted to the video track’s matrix settings
1 parent11e2363 commit2d4eec1

File tree

6 files changed

+235
-29
lines changed

6 files changed

+235
-29
lines changed

‎.changeset/plain-bears-remain.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
'@webav/av-cliper':patch
3+
---
4+
5+
fix: MP4Clip has not adapted to the video track’s matrix settings

‎packages/av-cliper/src/clips/__tests__/mp4-clip.test.ts‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
importmp4box,{MP4ArrayBuffer}from'@webav/mp4box.js';
22
import{file,write}from'opfs-tools';
33
import{expect,test,vi}from'vitest';
4-
import{parseMatrix}from'../../mp4-utils/mp4box-utils';
54
import{MP4Clip}from'../mp4-clip';
65

76
constmp4_123=`//${location.host}/video/123.mp4`;
@@ -182,15 +181,6 @@ test('get file header data', async () => {
182181
);
183182

184183
expect(boxfile.moov?.mvhd.matrix.length).toBe(9);
185-
expect(parseMatrix(boxfile.moov?.mvhd.matrix!)).toEqual({
186-
perspective:1,
187-
rotationDeg:0,
188-
rotationRad:0,
189-
scaleX:1,
190-
scaleY:1,
191-
translateX:0,
192-
translateY:0,
193-
});
194184
});
195185

196186
test('decode incorrectFrameTypeMp4',async()=>{

‎packages/av-cliper/src/clips/mp4-clip.ts‎

Lines changed: 60 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { MP4Info, MP4Sample } from '@webav/mp4box.js';
33
import{file,tmpfile,write}from'opfs-tools';
44
import{audioResample,extractPCM4AudioData,sleep}from'../av-utils';
55
import{
6+
createVFRotater,
67
extractFileConfig,
78
parseMatrix,
89
quickParseMP4File,
@@ -108,13 +109,14 @@ export class MP4Clip implements IClip {
108109
/**存储视频平移旋转信息，目前只还原旋转 */
109110
#parsedMatrix={
110111
perspective:1,
111-
rotationDeg:0,
112112
rotationRad:0,
113+
rotationDeg:0,
113114
scaleX:1,
114115
scaleY:1,
115116
translateX:0,
116117
translateY:0,
117118
};
119+
#vfRotater:(vf:VideoFrame|null)=>VideoFrame|null=(vf)=>vf;
118120

119121
#volume=1;
120122

@@ -206,7 +208,22 @@ export class MP4Clip implements IClip {
206208
this.#videoFrameFinder=videoFrameFinder;
207209
this.#audioFrameFinder=audioFrameFinder;
208210

209-
this.#meta=genMeta(decoderConf,videoSamples,audioSamples);
211+
const{ codedWidth, codedHeight}=decoderConf.video??{};
212+
if(codedWidth&&codedHeight){
213+
this.#vfRotater=createVFRotater(
214+
codedWidth,
215+
codedHeight,
216+
parsedMatrix.rotationDeg,
217+
);
218+
}
219+
220+
this.#meta=genMeta(
221+
decoderConf,
222+
videoSamples,
223+
audioSamples,
224+
parsedMatrix.rotationDeg,
225+
);
226+
210227
this.#log.info('MP4Clip meta:',this.#meta);
211228
return{ ...this.#meta};
212229
},
@@ -243,7 +260,7 @@ export class MP4Clip implements IClip {
243260

244261
const[audio,video]=awaitPromise.all([
245262
this.#audioFrameFinder?.find(time)??[],
246-
this.#videoFrameFinder?.find(time),
263+
this.#videoFrameFinder?.find(time).then(this.#vfRotater),
247264
]);
248265

249266
if(video==null){
@@ -476,6 +493,7 @@ function genMeta(
476493
decoderConf:MP4DecoderConf,
477494
videoSamples:ExtMP4Sample[],
478495
audioSamples:ExtMP4Sample[],
496+
rotationDeg:number,
479497
){
480498
constmeta={
481499
duration:0,
@@ -487,6 +505,11 @@ function genMeta(
487505
if(decoderConf.video!=null&&videoSamples.length>0){
488506
meta.width=decoderConf.video.codedWidth??0;
489507
meta.height=decoderConf.video.codedHeight??0;
508+
// 90, 270 度，需要交换宽高
509+
constnormalizedRotation=(Math.round(rotationDeg/90)*90+360)%360;
510+
if(normalizedRotation===90||normalizedRotation===270){
511+
[meta.width,meta.height]=[meta.height,meta.width];
512+
}
490513
}
491514
if(decoderConf.audio!=null&&audioSamples.length>0){
492515
meta.audioSampleRate=DEFAULT_AUDIO_CONF.sampleRate;
@@ -551,8 +574,8 @@ async function mp4FileToSamples(otFile: OPFSToolFile, opts: IMP4ClipOpts = {}) {
551574
letheaderBoxPos:Array<{start:number;size:number}>=[];
552575
constparsedMatrix={
553576
perspective:1,
554-
rotationDeg:0,
555577
rotationRad:0,
578+
rotationDeg:0,
556579
scaleX:1,
557580
scaleY:1,
558581
translateX:0,
@@ -571,7 +594,7 @@ async function mp4FileToSamples(otFile: OPFSToolFile, opts: IMP4ClipOpts = {}) {
571594
constmoov=data.mp4boxFile.moov!;
572595
headerBoxPos.push({start:moov.start,size:moov.size});
573596

574-
Object.assign(parsedMatrix,parseMatrix(moov.mvhd.matrix));
597+
Object.assign(parsedMatrix,parseMatrix(mp4Info.videoTracks[0]?.matrix));
575598

576599
let{videoDecoderConf:vc,audioDecoderConf:ac}=extractFileConfig(
577600
data.mp4boxFile,
@@ -1562,4 +1585,36 @@ if (import.meta.vitest) {
15621585
expect(normalized.size).toBe(1000);
15631586
expect(normalized.is_sync).toBe(normalized.is_idr);
15641587
});
1588+
1589+
it('genMeta adjusts width and height based on rotation',()=>{
1590+
constmeta=genMeta(
1591+
{
1592+
video:{
1593+
codedWidth:1920,
1594+
codedHeight:1080,
1595+
},
1596+
audio:null,
1597+
}asany,
1598+
[{cts:0,duration:1000}]asany,
1599+
[],
1600+
90,
1601+
);
1602+
expect(meta.width).toBe(1080);
1603+
expect(meta.height).toBe(1920);
1604+
1605+
constmeta2=genMeta(
1606+
{
1607+
video:{
1608+
codedWidth:1920,
1609+
codedHeight:1080,
1610+
},
1611+
audio:null,
1612+
}asany,
1613+
[{cts:0,duration:1000}]asany,
1614+
[],
1615+
180,
1616+
);
1617+
expect(meta2.width).toBe(1920);
1618+
expect(meta2.height).toBe(1080);
1619+
});
15651620
}

‎packages/av-cliper/src/mp4-utils/__tests__/mp4-utils.test.ts‎

Lines changed: 122 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
1-
import{beforeAll,describe,expect,test,vi}from'vitest';
2-
importmp4boxfrom'@webav/mp4box.js';
31
import{autoReadStream,file2stream}from'@webav/internal-utils';
2+
importmp4boxfrom'@webav/mp4box.js';
43
import{file,write}from'opfs-tools';
5-
import{quickParseMP4File}from'../mp4box-utils';
4+
import{beforeAll,describe,expect,test,vi}from'vitest';
5+
import{
6+
createVFRotater,
7+
parseMatrix,
8+
quickParseMP4File,
9+
}from'../mp4box-utils';
610

711
beforeAll(()=>{
812
vi.useFakeTimers();
@@ -95,3 +99,118 @@ test('quickParseMP4File', async () => {
9599
expect(sampleCount).toBe(40);
96100
awaitreader.close();
97101
});
102+
103+
test('vfRotater can be rotate VideoFrame instance',()=>{
104+
constvf=newVideoFrame(newUint8Array(200*100*4),{
105+
codedHeight:100,
106+
codedWidth:200,
107+
format:'RGBA',
108+
timestamp:0,
109+
});
110+
111+
// Test 90 degree rotation
112+
constrotater90=createVFRotater(200,100,90);
113+
constrotatedVF90=rotater90(vf.clone());
114+
expect(rotatedVF90).not.toBeNull();
115+
if(rotatedVF90==null)thrownewError('must not be null');
116+
expect(rotatedVF90.codedWidth).toBe(100);
117+
expect(rotatedVF90.codedHeight).toBe(200);
118+
rotatedVF90.close();
119+
120+
// Test 180 degree rotation
121+
constrotater180=createVFRotater(200,100,180);
122+
constrotatedVF180=rotater180(vf.clone());
123+
expect(rotatedVF180).not.toBeNull();
124+
if(rotatedVF180==null)thrownewError('must not be null');
125+
expect(rotatedVF180.codedWidth).toBe(200);
126+
expect(rotatedVF180.codedHeight).toBe(100);
127+
rotatedVF180.close();
128+
129+
// Test 270 degree rotation
130+
constrotater270=createVFRotater(200,100,270);
131+
constrotatedVF270=rotater270(vf.clone());
132+
expect(rotatedVF270).not.toBeNull();
133+
if(rotatedVF270==null)thrownewError('must not be null');
134+
expect(rotatedVF270.codedWidth).toBe(100);
135+
expect(rotatedVF270.codedHeight).toBe(200);
136+
rotatedVF270.close();
137+
138+
// Test 0 degree rotation
139+
constrotater0=createVFRotater(200,100,0);
140+
constvfClone=vf.clone();
141+
constrotatedVF0=rotater0(vfClone);
142+
// For 0 rotation, it should return the original frame
143+
expect(rotatedVF0).toBe(vfClone);
144+
rotatedVF0?.close();
145+
146+
vf.close();
147+
});
148+
149+
describe('parseMatrix',()=>{
150+
test('should throw error for invalid matrix length',()=>{
151+
constmatrix=newInt32Array(8);
152+
expect(parseMatrix(matrix)).toEqual({});
153+
});
154+
155+
test('should parse 0 degree rotation matrix',()=>{
156+
constmatrix=newInt32Array([65536,0,0,0,65536,0,0,0,1073741824]);
157+
constresult=parseMatrix(matrix);
158+
expect(result.rotationDeg).toBe(0);
159+
expect(result.scaleX).toBe(1);
160+
expect(result.scaleY).toBe(1);
161+
expect(result.translateX).toBe(0);
162+
expect(result.translateY).toBe(0);
163+
});
164+
165+
test('should parse 90 degree rotation matrix',()=>{
166+
// matrix for 90 deg rotation
167+
constmatrix=newInt32Array([
168+
0,65536,0,-65536,0,0,0,0,1073741824,
169+
]);
170+
constresult=parseMatrix(matrix);
171+
expect(result.rotationDeg).toBe(-90);
172+
expect(result.scaleX).toBe(1);
173+
expect(result.scaleY).toBe(1);
174+
});
175+
176+
test('should parse 180 degree rotation matrix',()=>{
177+
constmatrix=newInt32Array([
178+
-65536,0,0,0,-65536,0,0,0,1073741824,
179+
]);
180+
constresult=parseMatrix(matrix);
181+
expect(result.rotationDeg).toBe(180);
182+
expect(result.scaleX).toBe(1);
183+
expect(result.scaleY).toBe(1);
184+
});
185+
186+
test('should parse 270 degree rotation matrix',()=>{
187+
constmatrix=newInt32Array([
188+
0,-65536,0,65536,0,0,0,0,1073741824,
189+
]);
190+
constresult=parseMatrix(matrix);
191+
expect(result.rotationDeg).toBe(90);
192+
expect(result.scaleX).toBe(1);
193+
expect(result.scaleY).toBe(1);
194+
});
195+
196+
test('should parse matrix with translation',()=>{
197+
constwidth=1920;
198+
constheight=1080;
199+
// 180 deg rotation + translation
200+
constmatrix=newInt32Array([
201+
-65536,
202+
0,
203+
0,
204+
0,
205+
-65536,
206+
0,
207+
width*65536,
208+
height*65536,
209+
1073741824,
210+
]);
211+
constresult=parseMatrix(matrix);
212+
expect(result.rotationDeg).toBe(180);
213+
expect(result.translateX).toBe(width);
214+
expect(result.translateY).toBe(height);
215+
});
216+
});

‎packages/av-cliper/src/mp4-utils/mp4box-utils.ts‎

Lines changed: 47 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -180,19 +180,19 @@ export async function quickParseMP4File(
180180
}
181181
}
182182

183-
exportfunctionparseMatrix(matrix:Uint32Array){
184-
if(matrix.length!==9){
185-
thrownewError('Matrix must have 9 elements');
186-
}
183+
exportfunctionparseMatrix(matrix?:Int32Array){
184+
if(matrix?.length!==9)return{};
185+
186+
constsignedMatrix=newInt32Array(matrix.buffer);
187187

188188
// 提取并转成浮点数
189-
consta=matrix[0]/65536.0;
190-
constb=matrix[1]/65536.0;
191-
constc=matrix[3]/65536.0;
192-
constd=matrix[4]/65536.0;
193-
consttx=matrix[6]/65536.0;// 一般是 0
194-
constty=matrix[7]/65536.0;// 一般是 0
195-
constw=matrix[8]/(1<<30);// 一般是 1
189+
consta=signedMatrix[0]/65536.0;
190+
constb=signedMatrix[1]/65536.0;
191+
constc=signedMatrix[3]/65536.0;
192+
constd=signedMatrix[4]/65536.0;
193+
consttx=signedMatrix[6]/65536.0;// 一般是 0
194+
constty=signedMatrix[7]/65536.0;// 一般是 0
195+
constw=signedMatrix[8]/(1<<30);// 一般是 1
196196

197197
// 缩放
198198
constscaleX=Math.sqrt(a*a+c*c);
@@ -212,3 +212,39 @@ export function parseMatrix(matrix: Uint32Array) {
212212
perspective:w,
213213
};
214214
}
215+
216+
/**
217+
* 旋转 VideoFrame
218+
*/
219+
exportfunctioncreateVFRotater(
220+
width:number,
221+
height:number,
222+
rotationDeg:number,
223+
){
224+
constnormalizedRotation=(Math.round(rotationDeg/90)*90+360)%360;
225+
if(normalizedRotation===0)return(vf:VideoFrame|null)=>vf;
226+
227+
constrotatedWidth=
228+
normalizedRotation===90||normalizedRotation===270 ?height :width;
229+
constrotatedHeight=
230+
normalizedRotation===90||normalizedRotation===270 ?width :height;
231+
232+
constcanvas=newOffscreenCanvas(rotatedWidth,rotatedHeight);
233+
constctx=canvas.getContext('2d')!;
234+
235+
ctx.translate(rotatedWidth/2,rotatedHeight/2);
236+
ctx.rotate((normalizedRotation*Math.PI)/180);
237+
ctx.translate(-width/2,-height/2);
238+
239+
return(vf:VideoFrame|null)=>{
240+
if(vf==null)returnnull;
241+
242+
ctx.drawImage(vf,0,0);
243+
constnewVF=newVideoFrame(canvas,{
244+
timestamp:vf.timestamp,
245+
duration:vf.duration??0,
246+
});
247+
vf.close();
248+
returnnewVF;
249+
};
250+
}

‎types/mp4box.d.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ declare module '@webav/mp4box.js' {
1818
}
1919

2020
exportinterfaceMP4VideoTrackextendsMP4MediaTrack{
21+
matrix:Int32Array;
2122
video:{
2223
width:number;
2324
height:number;

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp