Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Speed up test setup #118

Merged
merged 6 commits into from
Jul 23, 2024
Merged

Speed up test setup #118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0661775
configure separate test for provisioning normally
Barabas5532 Jul 20, 2024
3d2dd75
Add integration test for provisioning
Barabas5532 Jul 20, 2024
97c35d0
clean up logging
Barabas5532 Jul 20, 2024
52e6d59
Update API test
Barabas5532 Jul 20, 2024
50f447e
remove TODO
Barabas5532 Jul 20, 2024
42059f8
add license headers
Barabas5532 Jul 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
270 changes: 157 additions & 113 deletions frontend/integration_test/simple_test.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,15 @@
@Tags(['api'])
library;

import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:logging/logging.dart';
import 'package:shrapnel/main.dart';
import 'package:shrapnel/midi_mapping/model/models.dart';

import '../test/firmware_api_test/util.dart';
import '../test/home_page_object.dart';
import '../test/util.dart';

// preset test
// start up
Expand Down Expand Up @@ -82,8 +84,7 @@ const networkSsid = String.fromEnvironment('NETWORK_SSID');
const networkPassphrase = String.fromEnvironment('NETWORK_PASSPHRASE');
// ignore: do_not_use_environment
const firmwareBinaryPath = String.fromEnvironment('FIRMWARE_BINARY_PATH');
// ignore: do_not_use_environment
const useFastProvisioning = bool.fromEnvironment('FAST_PROVISIONING');
const port = '/dev/ttyUSB0';

final _log = Logger('test');

Expand All @@ -96,136 +97,179 @@ void main() {

setUpAll(() async {
macAddress = await flashFirmware(
port: port,
appPartitionAddress: 0x10000,
path: firmwareBinaryPath,
);
});

setUp(() async {
await nvsErase();
testWidgets('wifi provisioning', (tester) async {
await nvsErase(port: port);

uart = await ShrapnelUart.open(port);
addTearDown(() => uart.dispose());

await connectToDutAccessPoint(macAddress);

// mDNS seems to be slow, use the IP address directly for faster testing.
await tester.pumpWidget(
App(
normalHost: dutIpAddress,
provisioningHost: '192.168.4.1',
),
);

final homePage = HomePageObject(tester);
final provisioningPage = await homePage.openWifiProvisioningPage();

await provisioningPage.startProvisioning();

await pumpWaitingFor(
tester: tester,
predicate: () =>
provisioningPage.findScanCompletePage.evaluate().isNotEmpty,
timeout: const Duration(seconds: 10),
);

await provisioningPage.openAdvancedSetup();

await provisioningPage.enterSsid(networkSsid);
await provisioningPage.enterPassword(networkPassphrase);

await provisioningPage.submitAdvanced();

await pumpWaitingFor(
tester: tester,
predicate: () => provisioningPage.findSuccessPage.evaluate().isNotEmpty,
timeout: const Duration(seconds: 10),
);

await tester.tap(find.byType(BackButton));

await pumpWaitingFor(
tester: tester,
predicate: () => homePage.findHomePage.evaluate().isNotEmpty,
timeout: const Duration(seconds: 10),
);

await homePage.waitUntilConnected();
});

group('wifi already set up', () {
setUp(() async {
await nvsErase(port: port);

uart = await ShrapnelUart.open('/dev/ttyUSB0');
// TODO move this into the uart class, so it logs by itself even if there
// are no external log listeners
// ensure at least one listener so logging side effect always runs
uart.log.listen((_) {});
addTearDown(uart.dispose);
uart = await ShrapnelUart.open(port);
addTearDown(uart.dispose);

if (useFastProvisioning) {
_log.warning('Bypassing Wi-Fi provisioning to speed up test execution');
await uart.provisionWifi(ssid: networkSsid, password: networkPassphrase);
} else {
// TODO we can still speed up tests where provisioning must be tested:
// - export NVS partition to file after first time wifi provisioning is used
// - reload it in the setup for each following test. This resets the NVS
// partition without requiring a fresh wifi provisioning run for each
// test.
//
// Alternatively, create a new test group that doesn't run the fast wifi
// provisioning setup, and instead runs the slow wifi setup.

await connectToDutAccessPoint(macAddress);
await setUpWiFi(ssid: networkSsid, password: networkPassphrase);
}
// Wait for firmware to start server after getting provisioned
await Future<void>.delayed(const Duration(seconds: 10));
});

// Wait for firmware to start server after getting provisioned
await Future<void>.delayed(const Duration(seconds: 10));
});
testWidgets('uart console smoke', (tester) async {
await tester.runAsync(() async {
// At least one response line has to arrive within 1 second
final responseLogs = uart.log.first.timeout(const Duration(seconds: 1));

await uart.sendMidiMessage(
const MidiMessage.controlChange(
channel: 0,
control: 1,
value: 0x00,
),
);

await expectLater(responseLogs, completes);
});
});

testWidgets('uart console smoke', (tester) async {
await tester.runAsync(() async {
// At least one response line has to arrive within 1 second
final responseLogs = uart.log.first.timeout(const Duration(seconds: 1));
testWidgets('audio parameters', (tester) async {
await tester.runAsync(() async {
// audio parameter basic test
// factory reset
// all parameters loaded correctly
// update a parameter
// force save to NVS
// reboot
// check parameter reloaded
});
});

await uart.sendMidiMessage(
const MidiMessage.controlChange(
channel: 0,
control: 1,
value: 0x00,
testWidgets('simple test', (tester) async {
// mDNS seems to be slow, use the IP address directly for faster testing.
await tester.pumpWidget(
App(
normalHost: dutIpAddress,
provisioningHost: '192.168.4.1',
),
);

await expectLater(responseLogs, completes);
});
});
// wait until connected
// poll connection status widget until ready with timeout
final homePage = HomePageObject(tester);
await homePage.waitUntilConnected();

testWidgets('audio parameters', (tester) async {
await tester.runAsync(() async {
// audio parameter basic test
// factory reset
// all parameters loaded correctly
// update a parameter
// force save to NVS
// reboot
// check parameter reloaded
});
});
await homePage.createPreset('Preset 1');

testWidgets('simple test', (tester) async {
await tester.pumpWidget(App());
final midiMappingPage = await homePage.openMidiMapping();

// wait until connected
// poll connection status widget until ready with timeout
final homePage = HomePageObject(tester);
await homePage.waitUntilConnected();
expect(midiMappingPage.findPage(), findsOneWidget);
expect(midiMappingPage.findMappingRows(), findsNothing);

await homePage.createPreset('Preset 1');

final midiMappingPage = await homePage.openMidiMapping();

expect(midiMappingPage.findPage(), findsOneWidget);
expect(midiMappingPage.findMappingRows(), findsNothing);

// create a mapping
final midiMappingCreatePage = await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(1);
await midiMappingCreatePage.selectCcNumber(2);
await midiMappingCreatePage.selectMode(MidiMappingMode.parameter);
// XXX: There is a bug in flutter where the DropdownButton's popup menu is
// unreliable during tests: https://github.com/flutter/flutter/issues/82908
//
// Pick an arbitrary parameter here. The only criteria for selection is that
// it actually works during the test. This is more likely if something is
// picked from the top of the list.
await midiMappingCreatePage.selectParameter('Chorus: DEPTH');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsOneWidget);

await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(2);
await midiMappingCreatePage.selectCcNumber(3);
await midiMappingCreatePage.selectMode(MidiMappingMode.button);
await midiMappingCreatePage.selectPreset('Preset 1');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsNWidgets(2));

await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(3);
await midiMappingCreatePage.selectCcNumber(4);
await midiMappingCreatePage.selectMode(MidiMappingMode.toggle);
// XXX: There is a bug in flutter where the DropdownButton's popup menu is
// unreliable during tests: https://github.com/flutter/flutter/issues/82908
//
// Pick an arbitrary parameter here. The only criteria for selection is that
// it actually works during the test. This is more likely if something is
// picked from the top of the list.
await midiMappingCreatePage.selectParameter('Chorus: RATE');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsNWidgets(3));
// create a mapping
final midiMappingCreatePage = await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(1);
await midiMappingCreatePage.selectCcNumber(2);
await midiMappingCreatePage.selectMode(MidiMappingMode.parameter);
// XXX: There is a bug in flutter where the DropdownButton's popup menu is
// unreliable during tests: https://github.com/flutter/flutter/issues/82908
//
// Pick an arbitrary parameter here. The only criteria for selection is that
// it actually works during the test. This is more likely if something is
// picked from the top of the list.
await midiMappingCreatePage.selectParameter('Chorus: DEPTH');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsOneWidget);

await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(2);
await midiMappingCreatePage.selectCcNumber(3);
await midiMappingCreatePage.selectMode(MidiMappingMode.button);
await midiMappingCreatePage.selectPreset('Preset 1');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsNWidgets(2));

await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(3);
await midiMappingCreatePage.selectCcNumber(4);
await midiMappingCreatePage.selectMode(MidiMappingMode.toggle);
// XXX: There is a bug in flutter where the DropdownButton's popup menu is
// unreliable during tests: https://github.com/flutter/flutter/issues/82908
//
// Pick an arbitrary parameter here. The only criteria for selection is that
// it actually works during the test. This is more likely if something is
// picked from the top of the list.
await midiMappingCreatePage.selectParameter('Chorus: RATE');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsNWidgets(3));
});
});
}
10 changes: 8 additions & 2 deletions frontend/lib/main.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,15 @@ class App extends StatelessWidget {
PresetsRepositoryBase? presetsRepository,
ParameterService? parameterService,
SelectedPresetRepositoryBase? selectedPresetRepository,
String? normalHost,
String? provisioningHost,
}) {
provisioningHost ??= 'guitar-dsp.local';
normalHost ??= 'guitar-dsp.local';

websocket ??= RobustWebsocket(
uri: Uri.parse('http://guitar-dsp.local:8080/websocket'),
uri: Uri.parse('http://guitar-dsp.local:8080/websocket')
.replace(host: normalHost),
);
_websocket = websocket;
apiWebsocket ??= ApiWebsocket(websocket: websocket);
Expand All @@ -125,7 +131,7 @@ class App extends StatelessWidget {
_log.info('Creating provisioning connection');
return Provisioning(
security: Security1(pop: 'abcd1234'),
transport: TransportHTTP('guitar-dsp.local'),
transport: TransportHTTP(provisioningHost!),
);
},
);
Expand Down
2 changes: 1 addition & 1 deletion frontend/lib/robust_websocket.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ class RobustWebsocket extends ChangeNotifier
while (socket == null) {
try {
final client = HttpClient();
client.connectionTimeout = const Duration(seconds: 15);
client.connectionTimeout = const Duration(seconds: 5);
final request = await client.openUrl('GET', uri);

final nonce = <int>[];
Expand Down
1 change: 1 addition & 0 deletions frontend/lib/wifi_provisioning.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,7 @@ class _WifiScanningScreenState extends State<_WifiScanningScreen> {
child = Padding(
padding: const EdgeInsets.all(8),
child: Column(
key: const Key('wifi scan complete page'),
children: [
Expanded(
child: ListView.builder(
Expand Down
Loading
Loading