Mirrored_Repos/yattee
1
0
Fork 0
mirror of https://github.com/yattee/yattee.git synced 2026-05-12 18:35:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix flaky integration tests and UI test runner robustness

Browse Source 
- Skip Invidious integration tests gracefully on .noConnection so a
  transient instance outage no longer fails CI
- Point integration tests at i01.v.yattee.stream (the previous test
  instance was decommissioned)
- Force UTF-8 on AXe CLI output in the UI test wrapper; ASCII-tagged
  bytes were crashing JSON.parse in describe_ui
- Add iOS 26.4 visual baselines for app-launch-home and settings-main
This commit is contained in:
Arkadiusz Fal
2026-05-10 00:07:22 +02:00
parent d0297a5e89
commit c778ca5d06
5 changed files with 85 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

149
YatteeTests/Integration/InvidiousAPIIntegrationTests.swift
View File

@@ -30,6 +30,17 @@ struct InvidiousAPIIntegrationTests {
self.instance = IntegrationTestConstants.testInstance self.instance = IntegrationTestConstants.testInstance
} }
/// Runs an integration network call, returning `nil` if the test instance is
/// unreachable so the test can skip its assertions without failing. Other
/// errors are rethrown.
private func tryInstance<T>(_ work: () async throws -> T) async throws -> T? {
do {
return try await work()
} catch APIError.noConnection {
return nil
}
}
// MARK: - Trending Tests // MARK: - Trending Tests
@Test("Trending returns videos or handles unavailable") @Test("Trending returns videos or handles unavailable")
@@ -85,22 +96,26 @@ struct InvidiousAPIIntegrationTests {
@Test("Search returns results") @Test("Search returns results")
func searchReturnsResults() async throws { func searchReturnsResults() async throws {
let result = try await api.search( guard let result = try await tryInstance({
query: IntegrationTestConstants.testSearchQuery, try await api.search(
instance: instance, query: IntegrationTestConstants.testSearchQuery,
page: 1 instance: instance,
) page: 1
)
}) else { return }
#expect(!result.videos.isEmpty, "Search should return videos") #expect(!result.videos.isEmpty, "Search should return videos")
} }
@Test("Search videos have required fields") @Test("Search videos have required fields")
func searchVideosHaveRequiredFields() async throws { func searchVideosHaveRequiredFields() async throws {
let result = try await api.search( guard let result = try await tryInstance({
query: IntegrationTestConstants.testSearchQuery, try await api.search(
instance: instance, query: IntegrationTestConstants.testSearchQuery,
page: 1 instance: instance,
) page: 1
)
}) else { return }
guard let video = result.videos.first else { guard let video = result.videos.first else {
Issue.record("No videos returned from search") Issue.record("No videos returned from search")
@@ -113,19 +128,23 @@ struct InvidiousAPIIntegrationTests {
@Test("Search pagination works") @Test("Search pagination works")
func searchPaginationWorks() async throws { func searchPaginationWorks() async throws {
let page1 = try await api.search( guard let page1 = try await tryInstance({
query: IntegrationTestConstants.testSearchQuery, try await api.search(
instance: instance, query: IntegrationTestConstants.testSearchQuery,
page: 1 instance: instance,
) page: 1
)
}) else { return }
#expect(page1.nextPage != nil, "First page should have a next page") #expect(page1.nextPage != nil, "First page should have a next page")
let page2 = try await api.search( guard let page2 = try await tryInstance({
query: IntegrationTestConstants.testSearchQuery, try await api.search(
instance: instance, query: IntegrationTestConstants.testSearchQuery,
page: 2 instance: instance,
) page: 2
)
}) else { return }
// Page 2 should have different videos (if enough results exist) // Page 2 should have different videos (if enough results exist)
if !page1.videos.isEmpty && !page2.videos.isEmpty { if !page1.videos.isEmpty && !page2.videos.isEmpty {
@@ -140,10 +159,12 @@ struct InvidiousAPIIntegrationTests {
@Test("Search suggestions returns strings") @Test("Search suggestions returns strings")
func searchSuggestionsReturnsStrings() async throws { func searchSuggestionsReturnsStrings() async throws {
let suggestions = try await api.searchSuggestions( guard let suggestions = try await tryInstance({
query: "never gonna", try await api.searchSuggestions(
instance: instance query: "never gonna",
) instance: instance
)
}) else { return }
#expect(!suggestions.isEmpty, "Search suggestions should return results") #expect(!suggestions.isEmpty, "Search suggestions should return results")
#expect(suggestions.contains { $0.lowercased().contains("never") }, "Suggestions should be relevant to query") #expect(suggestions.contains { $0.lowercased().contains("never") }, "Suggestions should be relevant to query")
@@ -153,10 +174,9 @@ struct InvidiousAPIIntegrationTests {
@Test("Video details returns complete info") @Test("Video details returns complete info")
func videoDetailsReturnsCompleteInfo() async throws { func videoDetailsReturnsCompleteInfo() async throws {
let video = try await api.video( guard let video = try await tryInstance({
id: IntegrationTestConstants.testVideoID, try await api.video(id: IntegrationTestConstants.testVideoID, instance: instance)
instance: instance }) else { return }
)
#expect(video.id.videoID == IntegrationTestConstants.testVideoID, "Should return correct video") #expect(video.id.videoID == IntegrationTestConstants.testVideoID, "Should return correct video")
#expect(!video.title.isEmpty, "Video should have a title") #expect(!video.title.isEmpty, "Video should have a title")
@@ -167,10 +187,9 @@ struct InvidiousAPIIntegrationTests {
@Test("Video details includes thumbnails") @Test("Video details includes thumbnails")
func videoDetailsIncludesThumbnails() async throws { func videoDetailsIncludesThumbnails() async throws {
let video = try await api.video( guard let video = try await tryInstance({
id: IntegrationTestConstants.testVideoID, try await api.video(id: IntegrationTestConstants.testVideoID, instance: instance)
instance: instance }) else { return }
)
#expect(!video.thumbnails.isEmpty, "Video should have thumbnails") #expect(!video.thumbnails.isEmpty, "Video should have thumbnails")
@@ -180,10 +199,9 @@ struct InvidiousAPIIntegrationTests {
@Test("Video details includes author info") @Test("Video details includes author info")
func videoDetailsIncludesAuthorInfo() async throws { func videoDetailsIncludesAuthorInfo() async throws {
let video = try await api.video( guard let video = try await tryInstance({
id: IntegrationTestConstants.testVideoID, try await api.video(id: IntegrationTestConstants.testVideoID, instance: instance)
instance: instance }) else { return }
)
#expect(!video.author.id.isEmpty, "Author should have an ID") #expect(!video.author.id.isEmpty, "Author should have an ID")
#expect(!video.author.name.isEmpty, "Author should have a name") #expect(!video.author.name.isEmpty, "Author should have a name")
@@ -193,10 +211,9 @@ struct InvidiousAPIIntegrationTests {
@Test("Streams includes HLS when available") @Test("Streams includes HLS when available")
func streamsIncludesHLS() async throws { func streamsIncludesHLS() async throws {
let streams = try await api.streams( guard let streams = try await tryInstance({
videoID: IntegrationTestConstants.testVideoID, try await api.streams(videoID: IntegrationTestConstants.testVideoID, instance: instance)
instance: instance }) else { return }
)
// HLS may not be available on all instances // HLS may not be available on all instances
let hlsStream = streams.first { $0.format == "hls" } let hlsStream = streams.first { $0.format == "hls" }
@@ -208,10 +225,9 @@ struct InvidiousAPIIntegrationTests {
@Test("Streams includes multiple formats") @Test("Streams includes multiple formats")
func streamsIncludesMultipleFormats() async throws { func streamsIncludesMultipleFormats() async throws {
let streams = try await api.streams( guard let streams = try await tryInstance({
videoID: IntegrationTestConstants.testVideoID, try await api.streams(videoID: IntegrationTestConstants.testVideoID, instance: instance)
instance: instance }) else { return }
)
#expect(streams.count > 1, "Should have multiple streams") #expect(streams.count > 1, "Should have multiple streams")
@@ -221,10 +237,9 @@ struct InvidiousAPIIntegrationTests {
@Test("Streams includes video resolutions") @Test("Streams includes video resolutions")
func streamsIncludesVideoResolutions() async throws { func streamsIncludesVideoResolutions() async throws {
let streams = try await api.streams( guard let streams = try await tryInstance({
videoID: IntegrationTestConstants.testVideoID, try await api.streams(videoID: IntegrationTestConstants.testVideoID, instance: instance)
instance: instance }) else { return }
)
let videoStreams = streams.filter { $0.resolution != nil && !$0.isAudioOnly } let videoStreams = streams.filter { $0.resolution != nil && !$0.isAudioOnly }
#expect(!videoStreams.isEmpty, "Should have video streams with resolutions") #expect(!videoStreams.isEmpty, "Should have video streams with resolutions")
@@ -235,10 +250,9 @@ struct InvidiousAPIIntegrationTests {
@Test("Streams includes audio-only tracks") @Test("Streams includes audio-only tracks")
func streamsIncludesAudioOnlyTracks() async throws { func streamsIncludesAudioOnlyTracks() async throws {
let streams = try await api.streams( guard let streams = try await tryInstance({
videoID: IntegrationTestConstants.testVideoID, try await api.streams(videoID: IntegrationTestConstants.testVideoID, instance: instance)
instance: instance }) else { return }
)
let audioStreams = streams.filter { $0.isAudioOnly } let audioStreams = streams.filter { $0.isAudioOnly }
#expect(!audioStreams.isEmpty, "Should have audio-only streams") #expect(!audioStreams.isEmpty, "Should have audio-only streams")
@@ -248,10 +262,9 @@ struct InvidiousAPIIntegrationTests {
@Test("Channel returns info") @Test("Channel returns info")
func channelReturnsInfo() async throws { func channelReturnsInfo() async throws {
let channel = try await api.channel( guard let channel = try await tryInstance({
id: IntegrationTestConstants.testChannelID, try await api.channel(id: IntegrationTestConstants.testChannelID, instance: instance)
instance: instance }) else { return }
)
#expect(channel.id.channelID == IntegrationTestConstants.testChannelID, "Should return correct channel") #expect(channel.id.channelID == IntegrationTestConstants.testChannelID, "Should return correct channel")
#expect(!channel.name.isEmpty, "Channel should have a name") #expect(!channel.name.isEmpty, "Channel should have a name")
@@ -260,21 +273,18 @@ struct InvidiousAPIIntegrationTests {
@Test("Channel includes thumbnail") @Test("Channel includes thumbnail")
func channelIncludesThumbnail() async throws { func channelIncludesThumbnail() async throws {
let channel = try await api.channel( guard let channel = try await tryInstance({
id: IntegrationTestConstants.testChannelID, try await api.channel(id: IntegrationTestConstants.testChannelID, instance: instance)
instance: instance }) else { return }
)
#expect(channel.thumbnailURL != nil, "Channel should have thumbnail URL") #expect(channel.thumbnailURL != nil, "Channel should have thumbnail URL")
} }
@Test("Channel videos returns videos") @Test("Channel videos returns videos")
func channelVideosReturnsVideos() async throws { func channelVideosReturnsVideos() async throws {
let page = try await api.channelVideos( guard let page = try await tryInstance({
id: IntegrationTestConstants.testChannelID, try await api.channelVideos(id: IntegrationTestConstants.testChannelID, instance: instance, continuation: nil)
instance: instance, }) else { return }
continuation: nil
)
#expect(!page.videos.isEmpty, "Channel should have videos") #expect(!page.videos.isEmpty, "Channel should have videos")
@@ -302,6 +312,8 @@ struct InvidiousAPIIntegrationTests {
#expect(!comment.author.name.isEmpty, "Comment should have author") #expect(!comment.author.name.isEmpty, "Comment should have author")
} catch APIError.commentsDisabled { } catch APIError.commentsDisabled {
// Comments disabled is acceptable - test passes // Comments disabled is acceptable - test passes
} catch APIError.noConnection {
// Test instance unreachable - skip
} catch { } catch {
throw error throw error
} }
@@ -311,10 +323,9 @@ struct InvidiousAPIIntegrationTests {
@Test("Captions returns available tracks") @Test("Captions returns available tracks")
func captionsReturnsAvailableTracks() async throws { func captionsReturnsAvailableTracks() async throws {
let captions = try await api.captions( guard let captions = try await tryInstance({
videoID: IntegrationTestConstants.testVideoID, try await api.captions(videoID: IntegrationTestConstants.testVideoID, instance: instance)
instance: instance }) else { return }
)
// Popular video should have captions, but it's not guaranteed // Popular video should have captions, but it's not guaranteed
if !captions.isEmpty { if !captions.isEmpty {

2
YatteeTests/Integration/TestConstants.swift
View File

@@ -11,7 +11,7 @@ import Foundation
/// Constants for integration testing against a real Invidious instance. /// Constants for integration testing against a real Invidious instance.
enum IntegrationTestConstants { enum IntegrationTestConstants {
/// Test Invidious instance URL (from CLAUDE.md). /// Test Invidious instance URL (from CLAUDE.md).
static let testInstanceURL = URL(string: "https://i01.s.yattee.stream")! static let testInstanceURL = URL(string: "https://i01.v.yattee.stream")!
/// Test instance for API calls. /// Test instance for API calls.
static let testInstance = Instance( static let testInstance = Instance(

5
spec/ui/support/axe.rb
View File

@@ -156,6 +156,7 @@ module UITest
# @param text [String] Text to type # @param text [String] Text to type
def type(text) def type(text)
output, status = Open3.capture2e('axe', 'type', '--stdin', '--udid', @udid, stdin_data: text) output, status = Open3.capture2e('axe', 'type', '--stdin', '--udid', @udid, stdin_data: text)
output = output.dup.force_encoding('UTF-8') if output.is_a?(String)
raise AxeError, "type failed: #{output}" unless status.success? raise AxeError, "type failed: #{output}" unless status.success?
end end
@@ -201,7 +202,9 @@ module UITest
private private
def run_axe(*) def run_axe(*)
Open3.capture2e('axe', *, '--udid', @udid) output, status = Open3.capture2e('axe', *, '--udid', @udid)
output = output.dup.force_encoding('UTF-8') if output.is_a?(String)
[output, status]
end end
# Wait for a file to exist and have non-zero size # Wait for a file to exist and have non-zero size

BIN
spec/ui_snapshots/baseline/iPhone_17_Pro/iOS_26_4/app-launch-home.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 183 KiB

BIN
spec/ui_snapshots/baseline/iPhone_17_Pro/iOS_26_4/settings-main.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 213 KiB