diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ef348ef614a9652e8d02a3e6b0fff774b686638b..2dbc42c444a6a5cdeefd29b8d117c783f3ffd15e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -258,10 +258,8 @@ integration windows:
   - cmd /c mklink /D $target_path $current_path
   - cd $GODIR\gitlab-runner
   script:
-  - $packages = (go list ./... | findstr -v "\/vendor\/" | Out-String | %{$_ -replace "\r\n"," "})
-  - $testCommand = "go test $packages"
-  - Invoke-Expression $testCommand
-  - if ( $LASTEXITCODE -ne 0 ) { exit $LASTEXITCODE }
+  - $script = Join-Path -Path "ci" -ChildPath "test_windows.ps1"
+  - Invoke-Expression $script
   allow_failure: true
   only:
   - /.*/@gitlab-org/gitlab-runner
diff --git a/ci/test_windows.ps1 b/ci/test_windows.ps1
new file mode 100644
index 0000000000000000000000000000000000000000..28113be81ac536364ce48808a05c75868f7a8cc4
--- /dev/null
+++ b/ci/test_windows.ps1
@@ -0,0 +1,16 @@
+param([string]$testOpts = "")
+
+$failed = @()
+go list ./... | Select-String -Pattern "gitlab.com/gitlab-org/gitlab-runner/vendor/" -NotMatch | ForEach-Object {
+    $pkg = %{$_ -replace "\r\n", "" }
+    $testCmd = "go test $testOpts $pkg"
+    Invoke-Expression $testCmd
+    if ( $LASTEXITCODE -ne 0 ) { $failed += $pkg }
+}
+
+if ($failed.count -ne 0) {
+    Write-Host "Failed packages"
+    $failed | Out-String | Write-Host
+
+    exit 1
+}