diff --git a/packages/models/src/videos/video-include.enum.ts b/packages/models/src/videos/video-include.enum.ts index 72f3d0b14a4..267ba93b72c 100644 --- a/packages/models/src/videos/video-include.enum.ts +++ b/packages/models/src/videos/video-include.enum.ts @@ -6,7 +6,8 @@ export const VideoInclude = { FILES: 1 << 3, CAPTIONS: 1 << 4, SOURCE: 1 << 5, - AUTOMATIC_TAGS: 1 << 6 + AUTOMATIC_TAGS: 1 << 6, + TAGS: 1 << 7 } as const export type VideoIncludeType = typeof VideoInclude[keyof typeof VideoInclude] diff --git a/packages/server-commands/src/videos/videos-command.ts b/packages/server-commands/src/videos/videos-command.ts index ddafc0d3f51..9e5a71679fb 100644 --- a/packages/server-commands/src/videos/videos-command.ts +++ b/packages/server-commands/src/videos/videos-command.ts @@ -417,14 +417,17 @@ export class VideosCommand extends AbstractCommand { mode?: 'legacy' | 'resumable' // default legacy waitTorrentGeneration?: boolean // default true completedExpectedStatus?: HttpStatusCodeType + videoChannelId?: number } = {}) { - const { mode = 'legacy', waitTorrentGeneration = true } = options + const { mode = 'legacy', videoChannelId, waitTorrentGeneration = true } = options let defaultChannelId = 1 - try { - const { videoChannels } = await this.server.users.getMyInfo({ token: options.token }) - defaultChannelId = videoChannels[0].id - } catch (e) { /* empty */ } + if (!videoChannelId) { + try { + const { videoChannels } = await this.server.users.getMyInfo({ token: options.token }) + defaultChannelId = videoChannels[0].id + } catch (e) { /* empty */ } + } // Override default attributes const attributes = { @@ -432,7 +435,7 @@ export class VideosCommand extends AbstractCommand { category: 5, licence: 4, language: 'zh', - channelId: defaultChannelId, + channelId: videoChannelId || defaultChannelId, nsfw: true, waitTranscoding: false, description: 'my super description', diff --git a/packages/tests/src/misc-endpoints.ts b/packages/tests/src/misc-endpoints.ts index 4c163f59a5b..0dff6e0cebd 100644 --- a/packages/tests/src/misc-endpoints.ts +++ b/packages/tests/src/misc-endpoints.ts @@ -193,15 +193,33 @@ describe('Test misc endpoints', function () { it('Should add videos, channel and accounts and get sitemap', async function () { this.timeout(35000) - await server.videos.upload({ attributes: { name: 'video 1', nsfw: false } }) - await server.videos.upload({ attributes: { name: 'video 2', nsfw: false } }) - await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } }) + const { token: user1Token } = await server.users.generate('user1') + const { token: user2Token } = await server.users.generate('user2') + const { token: user3Token } = await server.users.generate('user3') - await server.channels.create({ attributes: { name: 'channel1', displayName: 'channel 1' } }) - await server.channels.create({ attributes: { name: 'channel2', displayName: 'channel 2' } }) + const { id: channel1Id } = await server.channels.create({ + attributes: { name: 'channel1', displayName: 'channel 1' }, + token: user1Token + }) + const { id: channel2Id } = await server.channels.create({ + attributes: { name: 'channel2', displayName: 'channel 2' }, + token: user2Token + }) + const { id: channel3Id } = await server.channels.create({ + attributes: { name: 'channel3', displayName: 'channel 3' }, + token: user3Token + }) - await server.users.create({ username: 'user1', password: 'password' }) - await server.users.create({ username: 'user2', password: 'password' }) + const { id: video1Id } = await server.videos.upload({ attributes: { name: 'video 1', nsfw: false }, videoChannelId: channel1Id }) + await server.videos.upload({ attributes: { name: 'video 2', nsfw: false }, videoChannelId: channel2Id }) + await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE }, videoChannelId: channel3Id }) + + await server.videos.update({ + id: video1Id, + attributes: { + tags: [ 'fish', 'chips' ] + } + }) const res = await makeGetRequest({ url: server.url, @@ -216,11 +234,25 @@ describe('Test misc endpoints', function () { expect(res.text).to.contain('video 2') expect(res.text).to.not.contain('video 3') + expect(res.text).to.match(/.*\.jpg<\/video:thumbnail_loc>/) + expect(res.text).to.match(/.*\.webm<\/video:content_loc>/) + expect(res.text).to.match(/.*\/videos\/embed\/.*<\/video:player_loc>/) + expect(res.text).to.match(/.*<\/video:duration>/) + expect(res.text).to.match(/0<\/video:rating>/) + expect(res.text).to.match(/0<\/video:view_count>/) + expect(res.text).to.match(/.*<\/video:publication_date>/) + expect(res.text).to.match(/fish<\/video:tag>/) + expect(res.text).to.match(/chips<\/video:tag>/) + expect(res.text).to.match(/channel 1<\/video:uploader>/) + expect(res.text).to.match(/NO<\/video:live>/) + expect(res.text).to.contain('' + server.url + '/c/channel1/videos') expect(res.text).to.contain('' + server.url + '/c/channel2/videos') + expect(res.text).to.not.contain('' + server.url + '/c/channel3/videos') expect(res.text).to.contain('' + server.url + '/a/user1/video-channels') expect(res.text).to.contain('' + server.url + '/a/user2/video-channels') + expect(res.text).to.not.contain('' + server.url + '/a/user3/video-channels') }) it('Should not fail with big title/description videos', async function () { diff --git a/server/core/controllers/sitemap.ts b/server/core/controllers/sitemap.ts index 42fb54f7c6b..dddc991e171 100644 --- a/server/core/controllers/sitemap.ts +++ b/server/core/controllers/sitemap.ts @@ -5,11 +5,11 @@ import { logger } from '@server/helpers/logger.js' import { getServerActor } from '@server/models/application/application.js' import { buildNSFWFilter } from '../helpers/express-utils.js' import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants.js' -import { apiRateLimiter, asyncMiddleware } from '../middlewares/index.js' -import { cacheRoute } from '../middlewares/cache/cache.js' +import { apiRateLimiter, asyncMiddleware, cacheRoute } from '../middlewares/index.js' import { AccountModel } from '../models/account/account.js' import { VideoModel } from '../models/video/video.js' import { VideoChannelModel } from '../models/video/video-channel.js' +import { VideoFileStream, VideoInclude } from '@peertube/peertube-models' const sitemapRouter = express.Router() @@ -73,32 +73,64 @@ async function getSitemapAccountUrls () { async function getSitemapLocalVideoUrls () { const serverActor = await getServerActor() - const { data } = await VideoModel.listForApi({ - start: 0, - count: undefined, - sort: 'createdAt', - displayOnlyForFollower: { - actorId: serverActor.id, - orLocalVideos: true - }, - isLocal: true, - nsfw: buildNSFWFilter(), - countVideos: false - }) + let acc: { url: string, video: any[] }[] = [] + + const chunkSize = 200 + let hasData = true + let i = 0 + + while (hasData && i < 1000) { + const { data } = await VideoModel.listForApi({ + start: chunkSize * i, + count: chunkSize, + sort: 'createdAt', + displayOnlyForFollower: { + actorId: serverActor.id, + orLocalVideos: true + }, + isLocal: true, + nsfw: buildNSFWFilter(), + countVideos: false, + include: VideoInclude.FILES | VideoInclude.TAGS + }) + + hasData = data.length !== 0 + i++ + + acc = acc.concat( + data.map(v => { + const contentLoc = v.getHLSPlaylist()?.getMasterPlaylistUrl(v) || + v.getMaxQualityFile(VideoFileStream.VIDEO)?.getFileUrl(v) || + v.getMaxQualityFile(VideoFileStream.AUDIO)?.getFileUrl(v) + + return { + url: WEBSERVER.URL + v.getWatchStaticPath(), + video: [ + { + // Sitemap title should be < 100 characters + 'title': truncate(v.name, { length: 100, omission: '...' }), + // Sitemap description should be < 2000 characters + 'description': truncate(v.description || v.name, { length: 2000, omission: '...' }), + 'player_loc': WEBSERVER.URL + v.getEmbedStaticPath(), + 'thumbnail_loc': WEBSERVER.URL + v.getMiniatureStaticPath(), + 'content_loc': contentLoc, + 'duration': v.duration, + 'view_count': v.views, + 'publication_date': v.publishedAt.toISOString(), + 'uploader': v.VideoChannel.getDisplayName(), + 'uploader:info': v.VideoChannel.getClientUrl(), + 'live': v.isLive ? 'YES' : 'NO', + 'family_friendly': v.nsfw ? 'NO' : 'YES', + 'rating': (v.likes * 5) / (v.likes + v.dislikes) || 0, // Rating is between 0.0 and 5.0 + 'tag': v.Tags.map(t => t.name) + } + ] + } + }) + ) + } - return data.map(v => ({ - url: WEBSERVER.URL + v.getWatchStaticPath(), - video: [ - { - // Sitemap title should be < 100 characters - title: truncate(v.name, { length: 100, omission: '...' }), - // Sitemap description should be < 2000 characters - description: truncate(v.description || v.name, { length: 2000, omission: '...' }), - player_loc: WEBSERVER.URL + v.getEmbedStaticPath(), - thumbnail_loc: WEBSERVER.URL + v.getMiniatureStaticPath() - } - ] - })) + return acc } function getSitemapBasicUrls () { diff --git a/server/core/models/account/account.ts b/server/core/models/account/account.ts index 9144a982377..8ae25aa9980 100644 --- a/server/core/models/account/account.ts +++ b/server/core/models/account/account.ts @@ -1,6 +1,6 @@ -import { Account, AccountSummary } from '@peertube/peertube-models' +import { Account, AccountSummary, VideoPrivacy } from '@peertube/peertube-models' import { ModelCache } from '@server/models/shared/model-cache.js' -import { FindOptions, IncludeOptions, Includeable, Op, Transaction, WhereOptions } from 'sequelize' +import { FindOptions, IncludeOptions, Includeable, Op, Transaction, WhereOptions, literal } from 'sequelize' import { AfterDestroy, AllowNull, @@ -422,7 +422,7 @@ export class AccountModel extends SequelizeModel { } static listLocalsForSitemap (sort: string): Promise { - const query = { + return AccountModel.unscoped().findAll({ attributes: [ ], offset: 0, order: getSort(sort), @@ -433,13 +433,19 @@ export class AccountModel extends SequelizeModel { where: { serverId: null } + }, + { + attributes: [ 'id' ], + model: VideoChannelModel.unscoped(), + required: true, + where: { + [Op.and]: [ + literal(`EXISTS (SELECT 1 FROM "video" WHERE "privacy" = ${VideoPrivacy.PUBLIC} AND "channelId" = "VideoChannels"."id")`) + ] + } } ] - } - - return AccountModel - .unscoped() - .findAll(query) + }) } toFormattedJSON (this: MAccountFormattable): Account { diff --git a/server/core/models/video/sql/video/shared/video-model-builder.ts b/server/core/models/video/sql/video/shared/video-model-builder.ts index cb182a38b0c..ead5b600c3a 100644 --- a/server/core/models/video/sql/video/shared/video-model-builder.ts +++ b/server/core/models/video/sql/video/shared/video-model-builder.ts @@ -121,6 +121,10 @@ export class VideoModelBuilder { if (include & VideoInclude.AUTOMATIC_TAGS) { this.addAutoTag(row, videoModel) } + + if (include & VideoInclude.TAGS) { + this.addTag(row, videoModel) + } } } diff --git a/server/core/models/video/sql/video/videos-model-list-query-builder.ts b/server/core/models/video/sql/video/videos-model-list-query-builder.ts index 1cbde1b76fd..d4697bedc5f 100644 --- a/server/core/models/video/sql/video/videos-model-list-query-builder.ts +++ b/server/core/models/video/sql/video/videos-model-list-query-builder.ts @@ -109,6 +109,10 @@ export class VideosModelListQueryBuilder extends AbstractVideoQueryBuilder { this.includeAutomaticTags(serverActor.Account.id) } + if (options.include & VideoInclude.TAGS) { + this.includeTags() + } + const select = this.buildSelect() this.query = `${select} FROM (${this.innerQuery}) AS "tmp" ${this.joins} ${this.innerSort}` diff --git a/server/core/models/video/video-channel.ts b/server/core/models/video/video-channel.ts index bb920c68b53..36e9a5d51b1 100644 --- a/server/core/models/video/video-channel.ts +++ b/server/core/models/video/video-channel.ts @@ -1,5 +1,5 @@ import { forceNumber, pick } from '@peertube/peertube-core-utils' -import { ActivityPubActor, VideoChannel, VideoChannelSummary } from '@peertube/peertube-models' +import { ActivityPubActor, VideoChannel, VideoChannelSummary, VideoPrivacy } from '@peertube/peertube-models' import { CONFIG } from '@server/initializers/config.js' import { InternalEventEmitter } from '@server/lib/internal-event-emitter.js' import { MAccountHost } from '@server/types/models/index.js' @@ -523,7 +523,12 @@ export class VideoChannelModel extends SequelizeModel { serverId: null } } - ] + ], + where: { + [Op.and]: [ + literal(`EXISTS (SELECT 1 FROM "video" WHERE "privacy" = ${VideoPrivacy.PUBLIC} AND "channelId" = "VideoChannelModel"."id")`) + ] + } } return VideoChannelModel