From 22444735e9cd16356a834ab9fdb418a5d6e8fab1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 23 May 2025 00:43:00 +0000 Subject: [PATCH 1/6] Initial plan for issue From 202c87621b0f72993bf70b6c2f3084f358b5c783 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 23 May 2025 00:59:34 +0000 Subject: [PATCH 2/6] Implement telemetry tracking for CLI test usage Co-authored-by: karthiknadig <3840081+karthiknadig@users.noreply.github.com> --- src/client/telemetry/constants.ts | 1 + src/client/telemetry/index.ts | 7 + .../codeExecution/terminalReplWatcher.ts | 25 ++- .../terminalReplWatcher.unit.test.ts | 145 ++++++++++++++++++ 4 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts diff --git a/src/client/telemetry/constants.ts b/src/client/telemetry/constants.ts index ecc44177338a..33c250c30499 100644 --- a/src/client/telemetry/constants.ts +++ b/src/client/telemetry/constants.ts @@ -49,6 +49,7 @@ export enum EventName { UNITTEST_RUN = 'UNITTEST.RUN', UNITTEST_RUN_ALL_FAILED = 'UNITTEST.RUN_ALL_FAILED', UNITTEST_DISABLED = 'UNITTEST.DISABLED', + UNITTEST_RUN_CLI = 'UNITTEST.RUN.CLI', PYTHON_EXPERIMENTS_INIT_PERFORMANCE = 'PYTHON_EXPERIMENTS_INIT_PERFORMANCE', PYTHON_EXPERIMENTS_LSP_NOTEBOOKS = 'PYTHON_EXPERIMENTS_LSP_NOTEBOOKS', diff --git a/src/client/telemetry/index.ts b/src/client/telemetry/index.ts index 6c97bd083d96..359a9624891f 100644 --- a/src/client/telemetry/index.ts +++ b/src/client/telemetry/index.ts @@ -2174,6 +2174,13 @@ export interface IEventNamePropertyMapping { "unittest.disabled" : { "owner": "eleanorjboyd" } */ [EventName.UNITTEST_DISABLED]: never | undefined; + /** + * Telemetry event sent when a user runs tests via the CLI in terminal. + */ + /* __GDPR__ + "unittest.run.cli" : { "owner": "eleanorjboyd" } + */ + [EventName.UNITTEST_RUN_CLI]: never | undefined; /* Telemetry event sent to provide information on whether we have successfully identify the type of shell used. This information is useful in determining how well we identify shells on users machines. diff --git a/src/client/terminals/codeExecution/terminalReplWatcher.ts b/src/client/terminals/codeExecution/terminalReplWatcher.ts index 951961ab6901..1d5e0a8e95c1 100644 --- a/src/client/terminals/codeExecution/terminalReplWatcher.ts +++ b/src/client/terminals/codeExecution/terminalReplWatcher.ts @@ -3,8 +3,24 @@ import { onDidStartTerminalShellExecution } from '../../common/vscodeApis/window import { sendTelemetryEvent } from '../../telemetry'; import { EventName } from '../../telemetry/constants'; -function checkREPLCommand(command: string): undefined | 'manualTerminal' | `runningScript` { +function checkREPLCommand(command: string): undefined | 'manualTerminal' | `runningScript` | 'runningTest' { const lower = command.toLowerCase().trimStart(); + + // Check for test commands + if ( + lower.includes('pytest') || + (lower.startsWith('python') && lower.includes(' -m pytest')) || + (lower.startsWith('py ') && lower.includes(' -m pytest')) || + (lower.startsWith('python') && lower.includes(' -m unittest')) || + (lower.startsWith('py ') && lower.includes(' -m unittest')) || + (lower.startsWith('python') && lower.includes(' -m nose')) || + (lower.startsWith('py ') && lower.includes(' -m nose')) || + lower.includes('py.test') + ) { + return 'runningTest'; + } + + // Regular Python commands if (lower.startsWith('python') || lower.startsWith('py ')) { const parts = lower.split(' '); if (parts.length === 1) { @@ -20,7 +36,12 @@ export function registerTriggerForTerminalREPL(disposables: Disposable[]): void onDidStartTerminalShellExecution(async (e: TerminalShellExecutionStartEvent) => { const replType = checkREPLCommand(e.execution.commandLine.value); if (e.execution.commandLine.isTrusted && replType) { - sendTelemetryEvent(EventName.REPL, undefined, { replType }); + // Send test-specific telemetry if it's a test command + if (replType === 'runningTest') { + sendTelemetryEvent(EventName.UNITTEST_RUN_CLI); + } else { + sendTelemetryEvent(EventName.REPL, undefined, { replType }); + } } }), ); diff --git a/src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts b/src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts new file mode 100644 index 000000000000..c3fe6d61457f --- /dev/null +++ b/src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { expect } from 'chai'; +import * as sinon from 'sinon'; +import * as windowApis from '../../../client/common/vscodeApis/windowApis'; +import * as telemetryModule from '../../../client/telemetry'; +import { EventName } from '../../../client/telemetry/constants'; +import { registerTriggerForTerminalREPL } from '../../../client/terminals/codeExecution/terminalReplWatcher'; + +suite('Terminal REPL Watcher', () => { + let windowApisStub: sinon.SinonStub; + let telemetryStub: sinon.SinonStub; + + setup(() => { + windowApisStub = sinon.stub(windowApis, 'onDidStartTerminalShellExecution').returns({ + dispose: () => { + // Do nothing + } + }); + telemetryStub = sinon.stub(telemetryModule, 'sendTelemetryEvent'); + }); + + teardown(() => { + sinon.restore(); + }); + + test('Should send REPL telemetry when Python is invoked', () => { + windowApisStub.callsFake((callback) => { + callback({ + execution: { + commandLine: { + value: 'python script.py', + isTrusted: true + } + } + }); + return { + dispose: () => { + // Do nothing + } + }; + }); + + registerTriggerForTerminalREPL([]); + + expect(telemetryStub.calledOnce).to.be.true; + expect(telemetryStub.args[0][0]).to.equal(EventName.REPL); + expect(telemetryStub.args[0][2]).to.deep.equal({ replType: 'runningScript' }); + }); + + test('Should send unittest CLI telemetry when pytest is invoked', () => { + windowApisStub.callsFake((callback) => { + callback({ + execution: { + commandLine: { + value: 'python -m pytest', + isTrusted: true + } + } + }); + return { + dispose: () => { + // Do nothing + } + }; + }); + + registerTriggerForTerminalREPL([]); + + expect(telemetryStub.calledOnce).to.be.true; + expect(telemetryStub.args[0][0]).to.equal(EventName.UNITTEST_RUN_CLI); + }); + + test('Should send unittest CLI telemetry when unittest is invoked', () => { + windowApisStub.callsFake((callback) => { + callback({ + execution: { + commandLine: { + value: 'python -m unittest discover', + isTrusted: true + } + } + }); + return { + dispose: () => { + // Do nothing + } + }; + }); + + registerTriggerForTerminalREPL([]); + + expect(telemetryStub.calledOnce).to.be.true; + expect(telemetryStub.args[0][0]).to.equal(EventName.UNITTEST_RUN_CLI); + }); + + test('Should send unittest CLI telemetry when nose is invoked', () => { + windowApisStub.callsFake((callback) => { + callback({ + execution: { + commandLine: { + value: 'python -m nose', + isTrusted: true + } + } + }); + return { + dispose: () => { + // Do nothing + } + }; + }); + + registerTriggerForTerminalREPL([]); + + expect(telemetryStub.calledOnce).to.be.true; + expect(telemetryStub.args[0][0]).to.equal(EventName.UNITTEST_RUN_CLI); + }); + + test('Should send unittest CLI telemetry when py.test is invoked', () => { + windowApisStub.callsFake((callback) => { + callback({ + execution: { + commandLine: { + value: 'py.test', + isTrusted: true + } + } + }); + return { + dispose: () => { + // Do nothing + } + }; + }); + + registerTriggerForTerminalREPL([]); + + expect(telemetryStub.calledOnce).to.be.true; + expect(telemetryStub.args[0][0]).to.equal(EventName.UNITTEST_RUN_CLI); + }); +}); \ No newline at end of file From 097da2dde244a47e9720dccbdbaeb2cab8f52f9b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 23 May 2025 06:10:42 +0000 Subject: [PATCH 3/6] Remove nose test support Co-authored-by: karthiknadig <3840081+karthiknadig@users.noreply.github.com> --- .../codeExecution/terminalReplWatcher.ts | 2 -- .../terminalReplWatcher.unit.test.ts | 23 ------------------- 2 files changed, 25 deletions(-) diff --git a/src/client/terminals/codeExecution/terminalReplWatcher.ts b/src/client/terminals/codeExecution/terminalReplWatcher.ts index 1d5e0a8e95c1..59c2de4c897d 100644 --- a/src/client/terminals/codeExecution/terminalReplWatcher.ts +++ b/src/client/terminals/codeExecution/terminalReplWatcher.ts @@ -13,8 +13,6 @@ function checkREPLCommand(command: string): undefined | 'manualTerminal' | `runn (lower.startsWith('py ') && lower.includes(' -m pytest')) || (lower.startsWith('python') && lower.includes(' -m unittest')) || (lower.startsWith('py ') && lower.includes(' -m unittest')) || - (lower.startsWith('python') && lower.includes(' -m nose')) || - (lower.startsWith('py ') && lower.includes(' -m nose')) || lower.includes('py.test') ) { return 'runningTest'; diff --git a/src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts b/src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts index c3fe6d61457f..e5120db39931 100644 --- a/src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts +++ b/src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts @@ -97,29 +97,6 @@ suite('Terminal REPL Watcher', () => { expect(telemetryStub.args[0][0]).to.equal(EventName.UNITTEST_RUN_CLI); }); - test('Should send unittest CLI telemetry when nose is invoked', () => { - windowApisStub.callsFake((callback) => { - callback({ - execution: { - commandLine: { - value: 'python -m nose', - isTrusted: true - } - } - }); - return { - dispose: () => { - // Do nothing - } - }; - }); - - registerTriggerForTerminalREPL([]); - - expect(telemetryStub.calledOnce).to.be.true; - expect(telemetryStub.args[0][0]).to.equal(EventName.UNITTEST_RUN_CLI); - }); - test('Should send unittest CLI telemetry when py.test is invoked', () => { windowApisStub.callsFake((callback) => { callback({ From 56987729bd569198c62e64a4e1067d1c25af1312 Mon Sep 17 00:00:00 2001 From: eleanorjboyd <26030610+eleanorjboyd@users.noreply.github.com> Date: Tue, 27 May 2025 08:26:46 -0700 Subject: [PATCH 4/6] linting --- .../terminalReplWatcher.unit.test.ts | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts b/src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts index e5120db39931..0d84bb874b7a 100644 --- a/src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts +++ b/src/test/terminals/codeExecution/terminalReplWatcher.unit.test.ts @@ -18,7 +18,7 @@ suite('Terminal REPL Watcher', () => { windowApisStub = sinon.stub(windowApis, 'onDidStartTerminalShellExecution').returns({ dispose: () => { // Do nothing - } + }, }); telemetryStub = sinon.stub(telemetryModule, 'sendTelemetryEvent'); }); @@ -33,14 +33,14 @@ suite('Terminal REPL Watcher', () => { execution: { commandLine: { value: 'python script.py', - isTrusted: true - } - } + isTrusted: true, + }, + }, }); return { dispose: () => { // Do nothing - } + }, }; }); @@ -57,14 +57,14 @@ suite('Terminal REPL Watcher', () => { execution: { commandLine: { value: 'python -m pytest', - isTrusted: true - } - } + isTrusted: true, + }, + }, }); return { dispose: () => { // Do nothing - } + }, }; }); @@ -80,14 +80,14 @@ suite('Terminal REPL Watcher', () => { execution: { commandLine: { value: 'python -m unittest discover', - isTrusted: true - } - } + isTrusted: true, + }, + }, }); return { dispose: () => { // Do nothing - } + }, }; }); @@ -103,14 +103,14 @@ suite('Terminal REPL Watcher', () => { execution: { commandLine: { value: 'py.test', - isTrusted: true - } - } + isTrusted: true, + }, + }, }); return { dispose: () => { // Do nothing - } + }, }; }); @@ -119,4 +119,4 @@ suite('Terminal REPL Watcher', () => { expect(telemetryStub.calledOnce).to.be.true; expect(telemetryStub.args[0][0]).to.equal(EventName.UNITTEST_RUN_CLI); }); -}); \ No newline at end of file +}); From 8ed74f27c1fb7fac8700cf5d3ee26e73e1eebd15 Mon Sep 17 00:00:00 2001 From: eleanorjboyd <26030610+eleanorjboyd@users.noreply.github.com> Date: Tue, 27 May 2025 08:29:28 -0700 Subject: [PATCH 5/6] linting --- src/client/terminals/codeExecution/terminalReplWatcher.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/terminals/codeExecution/terminalReplWatcher.ts b/src/client/terminals/codeExecution/terminalReplWatcher.ts index 59c2de4c897d..3f59db575a18 100644 --- a/src/client/terminals/codeExecution/terminalReplWatcher.ts +++ b/src/client/terminals/codeExecution/terminalReplWatcher.ts @@ -5,11 +5,11 @@ import { EventName } from '../../telemetry/constants'; function checkREPLCommand(command: string): undefined | 'manualTerminal' | `runningScript` | 'runningTest' { const lower = command.toLowerCase().trimStart(); - + // Check for test commands if ( - lower.includes('pytest') || - (lower.startsWith('python') && lower.includes(' -m pytest')) || + lower.includes('pytest') || + (lower.startsWith('python') && lower.includes(' -m pytest')) || (lower.startsWith('py ') && lower.includes(' -m pytest')) || (lower.startsWith('python') && lower.includes(' -m unittest')) || (lower.startsWith('py ') && lower.includes(' -m unittest')) || @@ -17,7 +17,7 @@ function checkREPLCommand(command: string): undefined | 'manualTerminal' | `runn ) { return 'runningTest'; } - + // Regular Python commands if (lower.startsWith('python') || lower.startsWith('py ')) { const parts = lower.split(' '); From 6d2e4288c31d5d2c29a357c22025fb9bc414d024 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 27 May 2025 20:11:46 +0000 Subject: [PATCH 6/6] Consolidate conditionals for test command detection Co-authored-by: anthonykim1 <62267334+anthonykim1@users.noreply.github.com> --- src/client/terminals/codeExecution/terminalReplWatcher.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/client/terminals/codeExecution/terminalReplWatcher.ts b/src/client/terminals/codeExecution/terminalReplWatcher.ts index 3f59db575a18..47043ba1a59b 100644 --- a/src/client/terminals/codeExecution/terminalReplWatcher.ts +++ b/src/client/terminals/codeExecution/terminalReplWatcher.ts @@ -9,10 +9,8 @@ function checkREPLCommand(command: string): undefined | 'manualTerminal' | `runn // Check for test commands if ( lower.includes('pytest') || - (lower.startsWith('python') && lower.includes(' -m pytest')) || - (lower.startsWith('py ') && lower.includes(' -m pytest')) || - (lower.startsWith('python') && lower.includes(' -m unittest')) || - (lower.startsWith('py ') && lower.includes(' -m unittest')) || + (lower.startsWith('python') && (lower.includes(' -m pytest') || lower.includes(' -m unittest'))) || + (lower.startsWith('py ') && (lower.includes(' -m pytest') || lower.includes(' -m unittest'))) || lower.includes('py.test') ) { return 'runningTest';