From d9beef5b0a8253170bc422c6507f602ad32a4b44 Mon Sep 17 00:00:00 2001 From: tianyuan186 Date: Mon, 9 Nov 2015 14:12:30 -0800 Subject: [PATCH 1/5] Change parameters to run mutliple tests. --- Simulator/ExperimentalConfiguration.cs | 29 +++--- Simulator/Program.cs | 20 +++- Simulator/Simulator.cs | 130 +++++++++++++++---------- 3 files changed, 110 insertions(+), 69 deletions(-) diff --git a/Simulator/ExperimentalConfiguration.cs b/Simulator/ExperimentalConfiguration.cs index 1436c7c..06de342 100644 --- a/Simulator/ExperimentalConfiguration.cs +++ b/Simulator/ExperimentalConfiguration.cs @@ -22,15 +22,11 @@ public class BenignUserAccountGroup public string PasswordFrequencyFile = @"rockyou-withcount.txt"; - public ulong TotalLoginAttemptsToIssue = 10*Thousand; + public ulong TotalLoginAttemptsToIssue = 20; + public ulong RecordUnitAttempts = 10; - public double ChanceOfCoookieReUse = 0.90d; + public double ChanceOfCoookieReUse = 0.20d; public int MaxCookiesPerUserAccount = 10; - - public double ChanceOfIpReUse = 0.80d; - public int MaxIpPerUserAccount = 20; - - public double ChanceOfBenignPasswordTypo = 0.02d; public double ChanceOfBenignAccountNameTypoResultingInAValidUserName = 0.02d; public double ChanceOfAccidentallyUsingAnotherAccountPassword = 0.02d; @@ -39,10 +35,11 @@ public class BenignUserAccountGroup public double FractionOfLoginAttemptsFromAttacker = 0.5d; - public ulong NumberOfAttackerControlledAccounts = 1*Thousand; - - public uint NumberOfIpAddressesControlledByAttacker = 100;// * (uint)Thousand; - public double FractionOfMaliciousIPsToOverlapWithBenign = 0.1; + public ulong NumberOfAttackerControlledAccounts = 100*Thousand; + + public uint SizeOfBenignIpSpace = 300 * (uint)Thousand; + public uint SizeOfNonOverlappingAttackerIpSpace = 300 * (uint)Thousand; + public double FractionOfMaliciousIPsToOverlapWithBenign = 0.10d; public double ProbabilityThatAttackerChoosesAnInvalidAccount = 0.10d; @@ -51,15 +48,15 @@ public class BenignUserAccountGroup public readonly BenignUserAccountGroup[] BenignUserGroups = new BenignUserAccountGroup[] { // Group 0 logs in 5 times per day - new BenignUserAccountGroup() {GroupSize = 2*Thousand, LoginsPerYear = DaysPerYear*5}, + new BenignUserAccountGroup() {GroupSize = 200, LoginsPerYear = DaysPerYear*5}, // Group 1 logs in once per day - new BenignUserAccountGroup() {GroupSize = 2*Thousand, LoginsPerYear = DaysPerYear}, + new BenignUserAccountGroup() {GroupSize = 200, LoginsPerYear = DaysPerYear}, // Group 2 logs in once per week - new BenignUserAccountGroup() {GroupSize = 2*Thousand, LoginsPerYear = WeeksPerYear}, + new BenignUserAccountGroup() {GroupSize = 200, LoginsPerYear = WeeksPerYear}, // Group 3 logs in once per month - new BenignUserAccountGroup() {GroupSize = 2*Thousand, LoginsPerYear = MonthsPerYear}, + new BenignUserAccountGroup() {GroupSize = 200, LoginsPerYear = MonthsPerYear}, // Group 4 logs in once per year - new BenignUserAccountGroup() {GroupSize = 2*Thousand, LoginsPerYear = 1} + new BenignUserAccountGroup() {GroupSize = 200, LoginsPerYear = 1} }; } } diff --git a/Simulator/Program.cs b/Simulator/Program.cs index aeeedf6..b5e0970 100644 --- a/Simulator/Program.cs +++ b/Simulator/Program.cs @@ -10,10 +10,22 @@ public class Program { public async Task Main(string[] args) { - ExperimentalConfiguration expConfig = new ExperimentalConfiguration(); - BlockingAlgorithmOptions blockConfig = new BlockingAlgorithmOptions(); - Simulator simulator = new Simulator(expConfig, blockConfig); - await simulator.Run(); + for(int i=0; i<11; i++) + { + ExperimentalConfiguration expConfig1 = new ExperimentalConfiguration(); + BlockingAlgorithmOptions blockConfig1 = new BlockingAlgorithmOptions(); + blockConfig1.BlockThresholdUnpopularPassword = 100 + 200 * i; + Simulator simulator = new Simulator(expConfig1, blockConfig1); + Console.WriteLine("Unpopularpassword {0}", blockConfig1.BlockThresholdUnpopularPassword); + + await simulator.Run(blockConfig1); + } + + + //ExperimentalConfiguration expConfig = new ExperimentalConfiguration(); + //BlockingAlgorithmOptions blockConfig = new BlockingAlgorithmOptions(); + //Simulator simulator = new Simulator(expConfig, blockConfig); + //await simulator.Run(); } } } diff --git a/Simulator/Simulator.cs b/Simulator/Simulator.cs index 96404b8..599aef0 100644 --- a/Simulator/Simulator.cs +++ b/Simulator/Simulator.cs @@ -39,6 +39,11 @@ public class Stats public MemoryOnlyStableStore StableStore = new MemoryOnlyStableStore(); public ExperimentalConfiguration MyExperimentalConfiguration; + + + + + public Simulator(ExperimentalConfiguration myExperimentalConfiguration, BlockingAlgorithmOptions options = default(BlockingAlgorithmOptions)) { MyExperimentalConfiguration = myExperimentalConfiguration; @@ -86,7 +91,7 @@ public class Stats /// Evaluate the accuracy of our stopguessing service by sending user logins and malicious traffic /// /// - public async Task Run(CancellationToken cancellationToken = default(CancellationToken)) + public async Task Run(BlockingAlgorithmOptions options,CancellationToken cancellationToken = default(CancellationToken)) { //1.Create account from Rockyou //Create 2*accountnumber accounts, first half is benign accounts, and second half is correct accounts owned by attackers @@ -100,11 +105,9 @@ public class Stats allAccounts.AddRange(MaliciousAccounts); Parallel.ForEach(allAccounts, async simAccount => { - UserAccount account = UserAccount.Create(simAccount.UniqueId, (int)CreditLimits.Last().Limit, - simAccount.Password, "PBKDF2_SHA256", 1); - foreach (string cookie in simAccount.Cookies) - account.HashesOfDeviceCookiesThatHaveSuccessfullyLoggedIntoThisAccount.Add(LoginAttempt.HashCookie(cookie)); - await MyUserAccountController.PutAsync(account, cancellationToken: cancellationToken); + await MyUserAccountController.PutAsync( + UserAccount.Create(simAccount.UniqueId, (int)CreditLimits.Last().Limit, + simAccount.Password, "PBKDF2_SHA256", 1), cancellationToken: cancellationToken); }); } catch (Exception e) @@ -128,58 +131,87 @@ public class Stats sw.Start(); Stats stats = new Stats(); - int count = 0; -// List Runtime = new List(new int[MyExperimentalConfiguration.TotalLoginAttemptsToIssue]); - await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttemptsToIssue, async () => + ulong count = 0; + double falsePositiveRate = 0; + double falseNegativeRate = 0; + ulong bootstrapall = 0; + ulong bootstrapsuccess = 0; + // List Runtime = new List(new int[MyExperimentalConfiguration.TotalLoginAttemptsToIssue]); + + for (int bigi = 0; bigi <(int) (MyExperimentalConfiguration.TotalLoginAttemptsToIssue/MyExperimentalConfiguration.RecordUnitAttempts); bigi++) + { + + + + await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttemptsToIssue, async () => { - SimulatedLoginAttempt simAttempt; - if (StrongRandomNumberGenerator.GetFraction() < - MyExperimentalConfiguration.FractionOfLoginAttemptsFromAttacker) + SimulatedLoginAttempt simAttempt; + if (StrongRandomNumberGenerator.GetFraction() < + MyExperimentalConfiguration.FractionOfLoginAttemptsFromAttacker) + { + simAttempt = MaliciousLoginAttemptBreadthFirst(); + } + else + { + simAttempt = BenignLoginAttempt(); + } + + LoginAttempt attemptWithOutcome = await + MyLoginAttemptController.LocalPutAsync(simAttempt.Attempt, simAttempt.Password);//, + // cancellationToken: cancellationToken); + AuthenticationOutcome outcome = attemptWithOutcome.Outcome; + + lock (stats) + { + stats.TotalLoopIterationsThatShouldHaveRecordedStats++; + if (simAttempt.IsGuess) { - simAttempt = MaliciousLoginAttemptBreadthFirst(); + if (outcome == AuthenticationOutcome.CredentialsValidButBlocked) + stats.TruePositives++; + else if (outcome == AuthenticationOutcome.CredentialsValid) + stats.FalseNegatives++; + else + stats.GuessWasWrong++; } else { - simAttempt = BenignLoginAttempt(); - } - - LoginAttempt attemptWithOutcome = await - MyLoginAttemptController.LocalPutAsync(simAttempt.Attempt, simAttempt.Password);//, - // cancellationToken: cancellationToken); - AuthenticationOutcome outcome = attemptWithOutcome.Outcome; - - lock (stats) - { - stats.TotalLoopIterationsThatShouldHaveRecordedStats++; - if (simAttempt.IsGuess) - { - if (outcome == AuthenticationOutcome.CredentialsValidButBlocked) - stats.TruePositives++; - else if (outcome == AuthenticationOutcome.CredentialsValid) - stats.FalseNegatives++; - else - stats.GuessWasWrong++; - } + if (outcome == AuthenticationOutcome.CredentialsValid) + stats.TrueNegatives++; + else if (outcome == AuthenticationOutcome.CredentialsValidButBlocked) + stats.FalsePositives++; else - { - if (outcome == AuthenticationOutcome.CredentialsValid) - stats.TrueNegatives++; - else if (outcome == AuthenticationOutcome.CredentialsValidButBlocked) - stats.FalsePositives++; - else - stats.BenignErrors++; - } + stats.BenignErrors++; } + } }, - (e) => { - lock (stats) + (e) => + { + lock (stats) + { + stats.TotalExceptions++; + } + Console.Error.WriteLine(e.ToString()); + count++; + }); + + falsePositiveRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TruePositives); + falseNegativeRate = ((double)stats.FalseNegatives) / ((double)stats.FalseNegatives + stats.TrueNegatives); + using (StringWriter filename = new StringWriter()) + { + filename.Write("Detailed_Log_Unpopular{0}.txt", options.BlockThresholdUnpopularPassword); + + //string filename = "Detailed_Log_Unpopular{0}.txt", options.BlockThresholdUnpopularPassword; + using (StreamWriter detailed = File.AppendText(@filename.ToString())) { - stats.TotalExceptions++; + detailed.WriteLine("The false postive rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalsePositives, stats.TruePositives, falsePositiveRate * 100d); + detailed.WriteLine("The false negative rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalseNegatives, stats.TrueNegatives, falseNegativeRate * 100d); + } - Console.Error.WriteLine(e.ToString()); - count++; - }); + } + + } + sw.Stop(); @@ -187,8 +219,8 @@ await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttempt Console.WriteLine("Time Elapsed={0}", sw.Elapsed); Console.WriteLine("the new count is {0}", count); - double falsePositiveRate = ((double) stats.FalsePositives)/((double)stats.FalsePositives + stats.TruePositives); - double falseNegativeRate = ((double)stats.FalseNegatives)/((double)stats.FalseNegatives + stats.TrueNegatives); + falsePositiveRate = ((double) stats.FalsePositives)/((double)stats.FalsePositives + stats.TruePositives); + falseNegativeRate = ((double)stats.FalseNegatives)/((double)stats.FalseNegatives + stats.TrueNegatives); using (System.IO.StreamWriter file = new System.IO.StreamWriter(@"result_log.txt")) From ed603adaf529bbba7801cd8fb4fcc3b25f0123a4 Mon Sep 17 00:00:00 2001 From: tianyuan186 Date: Mon, 9 Nov 2015 14:38:02 -0800 Subject: [PATCH 2/5] Add detection rate and false detection rate. change experiment settings. --- Simulator/ExperimentalConfiguration.cs | 31 ++++++++++++++++---------- Simulator/Simulator.cs | 9 +++++++- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/Simulator/ExperimentalConfiguration.cs b/Simulator/ExperimentalConfiguration.cs index 06de342..f8c8a43 100644 --- a/Simulator/ExperimentalConfiguration.cs +++ b/Simulator/ExperimentalConfiguration.cs @@ -22,11 +22,17 @@ public class BenignUserAccountGroup public string PasswordFrequencyFile = @"rockyou-withcount.txt"; - public ulong TotalLoginAttemptsToIssue = 20; - public ulong RecordUnitAttempts = 10; + public ulong TotalLoginAttemptsToIssue = 1 * Million; + public ulong RecordUnitAttempts = 100*Thousand; - public double ChanceOfCoookieReUse = 0.20d; + + + public double ChanceOfCoookieReUse = 0.90d; public int MaxCookiesPerUserAccount = 10; + + public double ChanceOfIpReUse = 0.80d; + public int MaxIpPerUserAccount = 20; + public double ChanceOfBenignPasswordTypo = 0.02d; public double ChanceOfBenignAccountNameTypoResultingInAValidUserName = 0.02d; public double ChanceOfAccidentallyUsingAnotherAccountPassword = 0.02d; @@ -35,11 +41,12 @@ public class BenignUserAccountGroup public double FractionOfLoginAttemptsFromAttacker = 0.5d; - public ulong NumberOfAttackerControlledAccounts = 100*Thousand; + public ulong NumberOfAttackerControlledAccounts = 1*Thousand; + + public uint NumberOfIpAddressesControlledByAttacker = 100;// * (uint)Thousand; + public double FractionOfMaliciousIPsToOverlapWithBenign = 0.1; - public uint SizeOfBenignIpSpace = 300 * (uint)Thousand; - public uint SizeOfNonOverlappingAttackerIpSpace = 300 * (uint)Thousand; - public double FractionOfMaliciousIPsToOverlapWithBenign = 0.10d; + public double ProbabilityThatAttackerChoosesAnInvalidAccount = 0.10d; @@ -48,15 +55,15 @@ public class BenignUserAccountGroup public readonly BenignUserAccountGroup[] BenignUserGroups = new BenignUserAccountGroup[] { // Group 0 logs in 5 times per day - new BenignUserAccountGroup() {GroupSize = 200, LoginsPerYear = DaysPerYear*5}, + new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = DaysPerYear*5}, // Group 1 logs in once per day - new BenignUserAccountGroup() {GroupSize = 200, LoginsPerYear = DaysPerYear}, + new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = DaysPerYear}, // Group 2 logs in once per week - new BenignUserAccountGroup() {GroupSize = 200, LoginsPerYear = WeeksPerYear}, + new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = WeeksPerYear}, // Group 3 logs in once per month - new BenignUserAccountGroup() {GroupSize = 200, LoginsPerYear = MonthsPerYear}, + new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = MonthsPerYear}, // Group 4 logs in once per year - new BenignUserAccountGroup() {GroupSize = 200, LoginsPerYear = 1} + new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = 1} }; } } diff --git a/Simulator/Simulator.cs b/Simulator/Simulator.cs index 599aef0..a6b6861 100644 --- a/Simulator/Simulator.cs +++ b/Simulator/Simulator.cs @@ -135,6 +135,10 @@ await MyUserAccountController.PutAsync( ulong count = 0; double falsePositiveRate = 0; double falseNegativeRate = 0; + //The percentage of malicious attempts get caught (over all malicious attempts) + double detectionRate = 0; + //The percentage of benign attempts get labeled as malicious (over all benign attempts) + double falseDetectionRate = 0; ulong bootstrapall = 0; ulong bootstrapsuccess = 0; // List Runtime = new List(new int[MyExperimentalConfiguration.TotalLoginAttemptsToIssue]); @@ -197,6 +201,8 @@ await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttempt falsePositiveRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TruePositives); falseNegativeRate = ((double)stats.FalseNegatives) / ((double)stats.FalseNegatives + stats.TrueNegatives); + detectionRate = ((double)stats.TruePositives) / ((double)stats.TruePositives + stats.FalseNegatives); + falseDetectionRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TrueNegatives); using (StringWriter filename = new StringWriter()) { filename.Write("Detailed_Log_Unpopular{0}.txt", options.BlockThresholdUnpopularPassword); @@ -206,7 +212,8 @@ await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttempt { detailed.WriteLine("The false postive rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalsePositives, stats.TruePositives, falsePositiveRate * 100d); detailed.WriteLine("The false negative rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalseNegatives, stats.TrueNegatives, falseNegativeRate * 100d); - + detailed.WriteLine("The detection rate is {0}/({0}+{1}) ({2:F20}%)", stats.TruePositives, stats.FalseNegatives, detectionRate * 100d); + detailed.WriteLine("The false detection rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalsePositives, stats.TrueNegatives, falseDetectionRate * 100d); } } From 4c3106fcaf276ebd9704a687003970d91d0c63d3 Mon Sep 17 00:00:00 2001 From: tianyuan186 Date: Mon, 9 Nov 2015 15:10:15 -0800 Subject: [PATCH 3/5] Merge and add bootstrap counters --- Simulator/ExperimentalConfiguration.cs | 11 +-- Simulator/Simulator.cs | 128 +++++++++++++------------ 2 files changed, 71 insertions(+), 68 deletions(-) diff --git a/Simulator/ExperimentalConfiguration.cs b/Simulator/ExperimentalConfiguration.cs index f8c8a43..29030d8 100644 --- a/Simulator/ExperimentalConfiguration.cs +++ b/Simulator/ExperimentalConfiguration.cs @@ -22,10 +22,8 @@ public class BenignUserAccountGroup public string PasswordFrequencyFile = @"rockyou-withcount.txt"; - public ulong TotalLoginAttemptsToIssue = 1 * Million; - public ulong RecordUnitAttempts = 100*Thousand; - - + public ulong TotalLoginAttemptsToIssue = 1*Million; + public ulong RecordUnitAttempts = 100 * Thousand; public double ChanceOfCoookieReUse = 0.90d; public int MaxCookiesPerUserAccount = 10; @@ -33,6 +31,7 @@ public class BenignUserAccountGroup public double ChanceOfIpReUse = 0.80d; public int MaxIpPerUserAccount = 20; + public double ChanceOfBenignPasswordTypo = 0.02d; public double ChanceOfBenignAccountNameTypoResultingInAValidUserName = 0.02d; public double ChanceOfAccidentallyUsingAnotherAccountPassword = 0.02d; @@ -42,12 +41,10 @@ public class BenignUserAccountGroup public double FractionOfLoginAttemptsFromAttacker = 0.5d; public ulong NumberOfAttackerControlledAccounts = 1*Thousand; - + public uint NumberOfIpAddressesControlledByAttacker = 100;// * (uint)Thousand; public double FractionOfMaliciousIPsToOverlapWithBenign = 0.1; - - public double ProbabilityThatAttackerChoosesAnInvalidAccount = 0.10d; public uint NumberOfPopularPasswordsForAttackerToExploit = 1*(uint)Thousand; diff --git a/Simulator/Simulator.cs b/Simulator/Simulator.cs index a6b6861..66daae8 100644 --- a/Simulator/Simulator.cs +++ b/Simulator/Simulator.cs @@ -28,6 +28,8 @@ public class Stats public ulong TotalLoopIterations = 0; public ulong TotalExceptions = 0; public ulong TotalLoopIterationsThatShouldHaveRecordedStats = 0; + public ulong MaliciousCount = 0; + public ulong BenignCount = 0; } public IDistributedResponsibilitySet MyResponsibleHosts; @@ -39,11 +41,6 @@ public class Stats public MemoryOnlyStableStore StableStore = new MemoryOnlyStableStore(); public ExperimentalConfiguration MyExperimentalConfiguration; - - - - - public Simulator(ExperimentalConfiguration myExperimentalConfiguration, BlockingAlgorithmOptions options = default(BlockingAlgorithmOptions)) { MyExperimentalConfiguration = myExperimentalConfiguration; @@ -91,7 +88,7 @@ public class Stats /// Evaluate the accuracy of our stopguessing service by sending user logins and malicious traffic /// /// - public async Task Run(BlockingAlgorithmOptions options,CancellationToken cancellationToken = default(CancellationToken)) + public async Task Run(BlockingAlgorithmOptions options, CancellationToken cancellationToken = default(CancellationToken)) { //1.Create account from Rockyou //Create 2*accountnumber accounts, first half is benign accounts, and second half is correct accounts owned by attackers @@ -105,9 +102,11 @@ public class Stats allAccounts.AddRange(MaliciousAccounts); Parallel.ForEach(allAccounts, async simAccount => { - await MyUserAccountController.PutAsync( - UserAccount.Create(simAccount.UniqueId, (int)CreditLimits.Last().Limit, - simAccount.Password, "PBKDF2_SHA256", 1), cancellationToken: cancellationToken); + UserAccount account = UserAccount.Create(simAccount.UniqueId, (int)CreditLimits.Last().Limit, + simAccount.Password, "PBKDF2_SHA256", 1); + foreach (string cookie in simAccount.Cookies) + account.HashesOfDeviceCookiesThatHaveSuccessfullyLoggedIntoThisAccount.Add(LoginAttempt.HashCookie(cookie)); + await MyUserAccountController.PutAsync(account, cancellationToken: cancellationToken); }); } catch (Exception e) @@ -132,7 +131,8 @@ await MyUserAccountController.PutAsync( Stats stats = new Stats(); - ulong count = 0; + //ulong maliciouscount = 0; + //ulong benigncount = 0; double falsePositiveRate = 0; double falseNegativeRate = 0; //The percentage of malicious attempts get caught (over all malicious attempts) @@ -141,68 +141,74 @@ await MyUserAccountController.PutAsync( double falseDetectionRate = 0; ulong bootstrapall = 0; ulong bootstrapsuccess = 0; - // List Runtime = new List(new int[MyExperimentalConfiguration.TotalLoginAttemptsToIssue]); - for (int bigi = 0; bigi <(int) (MyExperimentalConfiguration.TotalLoginAttemptsToIssue/MyExperimentalConfiguration.RecordUnitAttempts); bigi++) - { + // List Runtime = new List(new int[MyExperimentalConfiguration.TotalLoginAttemptsToIssue]); + for (int bigi = 0; bigi < (int)(MyExperimentalConfiguration.TotalLoginAttemptsToIssue / MyExperimentalConfiguration.RecordUnitAttempts); bigi++) + { await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttemptsToIssue, async () => { - SimulatedLoginAttempt simAttempt; - if (StrongRandomNumberGenerator.GetFraction() < - MyExperimentalConfiguration.FractionOfLoginAttemptsFromAttacker) - { - simAttempt = MaliciousLoginAttemptBreadthFirst(); - } - else - { - simAttempt = BenignLoginAttempt(); - } - - LoginAttempt attemptWithOutcome = await - MyLoginAttemptController.LocalPutAsync(simAttempt.Attempt, simAttempt.Password);//, - // cancellationToken: cancellationToken); - AuthenticationOutcome outcome = attemptWithOutcome.Outcome; - - lock (stats) - { - stats.TotalLoopIterationsThatShouldHaveRecordedStats++; - if (simAttempt.IsGuess) + SimulatedLoginAttempt simAttempt; + if (StrongRandomNumberGenerator.GetFraction() < + MyExperimentalConfiguration.FractionOfLoginAttemptsFromAttacker) { - if (outcome == AuthenticationOutcome.CredentialsValidButBlocked) - stats.TruePositives++; - else if (outcome == AuthenticationOutcome.CredentialsValid) - stats.FalseNegatives++; - else - stats.GuessWasWrong++; + simAttempt = MaliciousLoginAttemptBreadthFirst(); } else { - if (outcome == AuthenticationOutcome.CredentialsValid) - stats.TrueNegatives++; - else if (outcome == AuthenticationOutcome.CredentialsValidButBlocked) - stats.FalsePositives++; + simAttempt = BenignLoginAttempt(); + } + + LoginAttempt attemptWithOutcome = await + MyLoginAttemptController.LocalPutAsync(simAttempt.Attempt, simAttempt.Password);//, + // cancellationToken: cancellationToken); + AuthenticationOutcome outcome = attemptWithOutcome.Outcome; + + lock (stats) + { + stats.TotalLoopIterationsThatShouldHaveRecordedStats++; + if(stats.TruePositives == 1) + { + bootstrapsuccess = stats.FalseNegatives; + bootstrapall = stats.MaliciousCount; + + } + if (simAttempt.IsGuess) + { + stats.MaliciousCount++; + if (outcome == AuthenticationOutcome.CredentialsValidButBlocked) + stats.TruePositives++; + else if (outcome == AuthenticationOutcome.CredentialsValid) + stats.FalseNegatives++; + else + stats.GuessWasWrong++; + } else - stats.BenignErrors++; + { + stats.BenignCount++; + if (outcome == AuthenticationOutcome.CredentialsValid) + stats.TrueNegatives++; + else if (outcome == AuthenticationOutcome.CredentialsValidButBlocked) + stats.FalsePositives++; + else + stats.BenignErrors++; + } } - } }, - (e) => - { - lock (stats) - { - stats.TotalExceptions++; - } - Console.Error.WriteLine(e.ToString()); - count++; + (e) => { + lock (stats) + { + stats.TotalExceptions++; + } + Console.Error.WriteLine(e.ToString()); + //count++; }); - falsePositiveRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TruePositives); falseNegativeRate = ((double)stats.FalseNegatives) / ((double)stats.FalseNegatives + stats.TrueNegatives); - detectionRate = ((double)stats.TruePositives) / ((double)stats.TruePositives + stats.FalseNegatives); - falseDetectionRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TrueNegatives); + detectionRate = ((double)stats.TruePositives) / ((double)stats.MaliciousCount); + falseDetectionRate = ((double)stats.FalsePositives) / ((double)stats.BenignCount); using (StringWriter filename = new StringWriter()) { filename.Write("Detailed_Log_Unpopular{0}.txt", options.BlockThresholdUnpopularPassword); @@ -212,8 +218,8 @@ await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttempt { detailed.WriteLine("The false postive rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalsePositives, stats.TruePositives, falsePositiveRate * 100d); detailed.WriteLine("The false negative rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalseNegatives, stats.TrueNegatives, falseNegativeRate * 100d); - detailed.WriteLine("The detection rate is {0}/({0}+{1}) ({2:F20}%)", stats.TruePositives, stats.FalseNegatives, detectionRate * 100d); - detailed.WriteLine("The false detection rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalsePositives, stats.TrueNegatives, falseDetectionRate * 100d); + detailed.WriteLine("The detection rate is {0}/({1}) ({2:F20}%)", stats.TruePositives, stats.MaliciousCount, detectionRate * 100d); + detailed.WriteLine("The false detection rate is {0}/({1}) ({2:F20}%)", stats.FalsePositives, stats.BenignCount, falseDetectionRate * 100d); } } @@ -224,10 +230,10 @@ await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttempt sw.Stop(); Console.WriteLine("Time Elapsed={0}", sw.Elapsed); - Console.WriteLine("the new count is {0}", count); + Console.WriteLine("the new count is {0}", stats.MaliciousCount+stats.BenignCount); - falsePositiveRate = ((double) stats.FalsePositives)/((double)stats.FalsePositives + stats.TruePositives); - falseNegativeRate = ((double)stats.FalseNegatives)/((double)stats.FalseNegatives + stats.TrueNegatives); + falsePositiveRate = ((double) stats.FalsePositives)/((double)stats.FalsePositives + stats.TruePositives); + falseNegativeRate = ((double)stats.FalseNegatives)/((double)stats.FalseNegatives + stats.TrueNegatives); using (System.IO.StreamWriter file = new System.IO.StreamWriter(@"result_log.txt")) From 34029a511ca3997055978ad539493185e60767bb Mon Sep 17 00:00:00 2001 From: tianyuan186 Date: Mon, 9 Nov 2015 15:41:26 -0800 Subject: [PATCH 4/5] bug fix --- Simulator/Simulator.cs | 52 ++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/Simulator/Simulator.cs b/Simulator/Simulator.cs index 66daae8..1f664c4 100644 --- a/Simulator/Simulator.cs +++ b/Simulator/Simulator.cs @@ -30,6 +30,8 @@ public class Stats public ulong TotalLoopIterationsThatShouldHaveRecordedStats = 0; public ulong MaliciousCount = 0; public ulong BenignCount = 0; + public ulong bootstrapall = 0; + public ulong bootstrapsuccess = 0; } public IDistributedResponsibilitySet MyResponsibleHosts; @@ -89,7 +91,7 @@ public class Stats /// /// public async Task Run(BlockingAlgorithmOptions options, CancellationToken cancellationToken = default(CancellationToken)) - { + { //1.Create account from Rockyou //Create 2*accountnumber accounts, first half is benign accounts, and second half is correct accounts owned by attackers @@ -115,7 +117,7 @@ public class Stats { file.WriteLine("{0} Exception caught in account creation.", e); file.WriteLine("time is {0}", DateTime.Now.ToString(CultureInfo.InvariantCulture)); -// file.WriteLine("How many requests? {0}", i); + // file.WriteLine("How many requests? {0}", i); } } @@ -139,8 +141,7 @@ public class Stats double detectionRate = 0; //The percentage of benign attempts get labeled as malicious (over all benign attempts) double falseDetectionRate = 0; - ulong bootstrapall = 0; - ulong bootstrapsuccess = 0; + // List Runtime = new List(new int[MyExperimentalConfiguration.TotalLoginAttemptsToIssue]); @@ -148,8 +149,8 @@ public class Stats for (int bigi = 0; bigi < (int)(MyExperimentalConfiguration.TotalLoginAttemptsToIssue / MyExperimentalConfiguration.RecordUnitAttempts); bigi++) { - await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttemptsToIssue, async () => - { + await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.RecordUnitAttempts, async () => + { SimulatedLoginAttempt simAttempt; if (StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.FractionOfLoginAttemptsFromAttacker) @@ -163,16 +164,16 @@ await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttempt LoginAttempt attemptWithOutcome = await MyLoginAttemptController.LocalPutAsync(simAttempt.Attempt, simAttempt.Password);//, - // cancellationToken: cancellationToken); + // cancellationToken: cancellationToken); AuthenticationOutcome outcome = attemptWithOutcome.Outcome; lock (stats) { stats.TotalLoopIterationsThatShouldHaveRecordedStats++; - if(stats.TruePositives == 1) - { - bootstrapsuccess = stats.FalseNegatives; - bootstrapall = stats.MaliciousCount; + if (stats.TruePositives == 1) + { + stats.bootstrapsuccess = stats.FalseNegatives; + stats.bootstrapall = stats.MaliciousCount; } if (simAttempt.IsGuess) @@ -196,19 +197,19 @@ await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttempt stats.BenignErrors++; } } - }, - (e) => { - lock (stats) - { - stats.TotalExceptions++; - } - Console.Error.WriteLine(e.ToString()); + }, + (e) => { + lock (stats) + { + stats.TotalExceptions++; + } + Console.Error.WriteLine(e.ToString()); //count++; }); falsePositiveRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TruePositives); falseNegativeRate = ((double)stats.FalseNegatives) / ((double)stats.FalseNegatives + stats.TrueNegatives); - detectionRate = ((double)stats.TruePositives) / ((double)stats.MaliciousCount); - falseDetectionRate = ((double)stats.FalsePositives) / ((double)stats.BenignCount); + detectionRate = ((double)stats.TruePositives) / ((double)stats.TruePositives + stats.FalseNegatives); + falseDetectionRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TrueNegatives); using (StringWriter filename = new StringWriter()) { filename.Write("Detailed_Log_Unpopular{0}.txt", options.BlockThresholdUnpopularPassword); @@ -218,8 +219,9 @@ await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttempt { detailed.WriteLine("The false postive rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalsePositives, stats.TruePositives, falsePositiveRate * 100d); detailed.WriteLine("The false negative rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalseNegatives, stats.TrueNegatives, falseNegativeRate * 100d); - detailed.WriteLine("The detection rate is {0}/({1}) ({2:F20}%)", stats.TruePositives, stats.MaliciousCount, detectionRate * 100d); - detailed.WriteLine("The false detection rate is {0}/({1}) ({2:F20}%)", stats.FalsePositives, stats.BenignCount, falseDetectionRate * 100d); + detailed.WriteLine("The detection rate is {0}/({0}+{1}) ({2:F20}%)", stats.TruePositives, stats.FalseNegatives, detectionRate * 100d); + detailed.WriteLine("The false detection rate is {0}/{0}+({1}) ({2:F20}%)", stats.FalsePositives, stats.TrueNegatives, falseDetectionRate * 100d); + detailed.WriteLine("Start to catch attacker after {0} requests and attackers have login {1} times into users accounts successfully.", stats.bootstrapall, stats.bootstrapsuccess); } } @@ -230,10 +232,10 @@ await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.TotalLoginAttempt sw.Stop(); Console.WriteLine("Time Elapsed={0}", sw.Elapsed); - Console.WriteLine("the new count is {0}", stats.MaliciousCount+stats.BenignCount); + Console.WriteLine("the new count is {0}", stats.MaliciousCount + stats.BenignCount); - falsePositiveRate = ((double) stats.FalsePositives)/((double)stats.FalsePositives + stats.TruePositives); - falseNegativeRate = ((double)stats.FalseNegatives)/((double)stats.FalseNegatives + stats.TrueNegatives); + falsePositiveRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TruePositives); + falseNegativeRate = ((double)stats.FalseNegatives) / ((double)stats.FalseNegatives + stats.TrueNegatives); using (System.IO.StreamWriter file = new System.IO.StreamWriter(@"result_log.txt")) From 99bd5e45d87f3ba4ba68d7d9d1827706a0a056ae Mon Sep 17 00:00:00 2001 From: tianyuan186 Date: Wed, 11 Nov 2015 01:43:03 -0800 Subject: [PATCH 5/5] Add parameter sweep, add misconfigured clients --- Simulator/ExperimentalConfiguration.cs | 25 +- Simulator/Program.cs | 225 +++++++++++++++++- Simulator/SimulatedAccount.cs | 1 + Simulator/Simulator.cs | 128 ++++++++-- Simulator/Simulator_LoginAttemptGeneration.cs | 159 +++++++++---- 5 files changed, 461 insertions(+), 77 deletions(-) diff --git a/Simulator/ExperimentalConfiguration.cs b/Simulator/ExperimentalConfiguration.cs index c4ebd99..a38ae2f 100644 --- a/Simulator/ExperimentalConfiguration.cs +++ b/Simulator/ExperimentalConfiguration.cs @@ -22,8 +22,8 @@ public class BenignUserAccountGroup public string PasswordFrequencyFile = @"rockyou-withcount.txt"; - public ulong TotalLoginAttemptsToIssue = 1*Million; - public ulong RecordUnitAttempts = 100 * Thousand; + public ulong TotalLoginAttemptsToIssue = 10*Thousand; + public ulong RecordUnitAttempts = Thousand; public double ChanceOfCoookieReUse = 0.90d; public int MaxCookiesPerUserAccount = 10; @@ -50,18 +50,35 @@ public class BenignUserAccountGroup public uint NumberOfPopularPasswordsForAttackerToExploit = 1*(uint)Thousand; + //public readonly BenignUserAccountGroup[] BenignUserGroups = new BenignUserAccountGroup[] + //{ + // // Group 0 logs in 5 times per day + // new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = DaysPerYear*5}, + // // Group 1 logs in once per day + // new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = DaysPerYear}, + // // Group 2 logs in once per week + // new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = WeeksPerYear}, + // // Group 3 logs in once per month + // new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = MonthsPerYear}, + // // Group 4 logs in once per year + // new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = 1} + //}; + + //Add clients behaviors public readonly BenignUserAccountGroup[] BenignUserGroups = new BenignUserAccountGroup[] { // Group 0 logs in 5 times per day new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = DaysPerYear*5}, // Group 1 logs in once per day - new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = DaysPerYear}, + new BenignUserAccountGroup() {GroupSize = 10*Thousand, LoginsPerYear = DaysPerYear}, // Group 2 logs in once per week new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = WeeksPerYear}, // Group 3 logs in once per month new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = MonthsPerYear}, // Group 4 logs in once per year - new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = 1} + new BenignUserAccountGroup() {GroupSize = 20*Thousand, LoginsPerYear = 1}, + //Group 5 use logs in 1 time a day, but they use clients that log in 100 times a day + new BenignUserAccountGroup() {GroupSize = 10*Thousand, LoginsPerYear = DaysPerYear*101 } }; } } diff --git a/Simulator/Program.cs b/Simulator/Program.cs index b5e0970..e5a5fd0 100644 --- a/Simulator/Program.cs +++ b/Simulator/Program.cs @@ -10,22 +10,237 @@ public class Program { public async Task Main(string[] args) { - for(int i=0; i<11; i++) + //for (int i = 1; i < 10; i++) + //{ + // ExperimentalConfiguration expConfig1 = new ExperimentalConfiguration(); + // BlockingAlgorithmOptions blockConfig1 = new BlockingAlgorithmOptions(); + // blockConfig1.FOR_SIMULATION_ONLY_TURN_ON_SSH_STUPID_MODE = true; + // //blockConfig1.BlockThresholdUnpopularPassword = 1 * i; + // //blockConfig1.BlockThresholdPopularPassword = blockConfig1.BlockThresholdUnpopularPassword; + // // + // // Industrial-best-practice baseline + // // + + // // Use the same threshold regardless of the popularity of the account password + // blockConfig1.BlockThresholdPopularPassword = + // blockConfig1.BlockThresholdUnpopularPassword =1*i; + // // Make all failures increase the count towards the threshold by one + // blockConfig1.PenaltyForInvalidAccount = + // blockConfig1.PenaltyMulitiplierForTypo = + // blockConfig1.BasePenaltyForInvalidPassword = + // 1d; + // // If the below is empty, the multiplier for any popularity level will be 1. + // blockConfig1.PenaltyForReachingEachPopularityThreshold = new List(); + // // Correct passwords shouldn't help + // blockConfig1.RewardForCorrectPasswordPerAccount = 0; + + // // + + + // expConfig1.TotalLoginAttemptsToIssue = 5000; + // expConfig1.RecordUnitAttempts = 5000; + // //expConfig1.ChanceOfBenignPasswordTypo = 0.2d; + // Simulator simulator = new Simulator(expConfig1, blockConfig1); + // Console.WriteLine("Unpopularpassword {0}", blockConfig1.BlockThresholdUnpopularPassword); + + // // await simulator.Run(blockConfig1); + //} + + ulong TotalLoginAttemptsToIssue = 200000; + ulong RecordUnitAttempts = 200000; + uint MaliciousIP = 2000; + //1. Vary BlockThresholdUnpopularPassword from 100 to 2100 (in steps of 200) + for (int i = 0; i < 11; i++) { ExperimentalConfiguration expConfig1 = new ExperimentalConfiguration(); BlockingAlgorithmOptions blockConfig1 = new BlockingAlgorithmOptions(); + //blockConfig1.RewardForCorrectPasswordPerAccount = -10 * i - 10; + //blockConfig1.PenaltyForInvalidAccount = 1 + (int)(i/3); + //blockConfig1.BlockThresholdPopularPassword = 20 + 10 * i; + //blockConfig1.BlockThresholdUnpopularPassword = 4*blockConfig1.BlockThresholdPopularPassword; blockConfig1.BlockThresholdUnpopularPassword = 100 + 200 * i; + expConfig1.TotalLoginAttemptsToIssue = TotalLoginAttemptsToIssue; + expConfig1.RecordUnitAttempts = RecordUnitAttempts; + expConfig1.NumberOfIpAddressesControlledByAttacker = MaliciousIP; + //blockConfig1.BlockThresholdPopularPassword = 60; + //blockConfig1.BlockThresholdUnpopularPassword = 900; + //blockConfig1.PenaltyForInvalidAccount = 35; + + + + + //blockConfig1.PenaltyForReachingEachPopularityThreshold = new List + //{ + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(100*1000d), Penalty = 10*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(10*1000d), Penalty = 20*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(1*1000d), Penalty = 25*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(100d), Penalty = 30*Math.Pow(2,0.3*i)}, + //}; Simulator simulator = new Simulator(expConfig1, blockConfig1); - Console.WriteLine("Unpopularpassword {0}", blockConfig1.BlockThresholdUnpopularPassword); - - await simulator.Run(blockConfig1); + Console.WriteLine("unpopularpassword {0}", blockConfig1.BlockThresholdUnpopularPassword); + + await simulator.Run(blockConfig1, "BlockThresholdUnpopularPassword"); } + //2. Vary blockthresholdpopularpassword from 20 - 120 and unpopularpassword is 4 times + + + + for (int i = 0; i < 10; i++) + { + ExperimentalConfiguration expConfig1 = new ExperimentalConfiguration(); + BlockingAlgorithmOptions blockConfig1 = new BlockingAlgorithmOptions(); + //blockConfig1.RewardForCorrectPasswordPerAccount = -10 * i - 10; + //blockConfig1.PenaltyForInvalidAccount = 1 + (int)(i/3); + blockConfig1.BlockThresholdPopularPassword = 20 + 10 * i; + blockConfig1.BlockThresholdUnpopularPassword = 4 * blockConfig1.BlockThresholdPopularPassword; + //blockConfig1.BlockThresholdUnpopularPassword = 100 + 200 * i; + expConfig1.TotalLoginAttemptsToIssue = TotalLoginAttemptsToIssue; + expConfig1.RecordUnitAttempts = RecordUnitAttempts; + expConfig1.NumberOfIpAddressesControlledByAttacker = 100; + //blockConfig1.BlockThresholdPopularPassword = 60; + //blockConfig1.BlockThresholdUnpopularPassword = 900; + //blockConfig1.PenaltyForInvalidAccount = 35; + + + + + //blockConfig1.PenaltyForReachingEachPopularityThreshold = new List + //{ + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(100*1000d), Penalty = 10*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(10*1000d), Penalty = 20*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(1*1000d), Penalty = 25*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(100d), Penalty = 30*Math.Pow(2,0.3*i)}, + //}; + Simulator simulator = new Simulator(expConfig1, blockConfig1); + Console.WriteLine("Popularpassword {0}", blockConfig1.BlockThresholdPopularPassword); + + await simulator.Run(blockConfig1, "BlockThresholdPopularPassword"); + } + + //3.Vary PenaltyForInvalidAccount from 1 to 10 (in steps of 1) + for (int i = 0; i < 10; i++) + { + ExperimentalConfiguration expConfig1 = new ExperimentalConfiguration(); + BlockingAlgorithmOptions blockConfig1 = new BlockingAlgorithmOptions(); + //blockConfig1.RewardForCorrectPasswordPerAccount = -10 * i - 10; + blockConfig1.PenaltyForInvalidAccount = 1 + i; + //blockConfig1.BlockThresholdPopularPassword = 20 + 10 * i; + //blockConfig1.BlockThresholdUnpopularPassword = 4 * blockConfig1.BlockThresholdPopularPassword; + //blockConfig1.BlockThresholdUnpopularPassword = 100 + 200 * i; + expConfig1.TotalLoginAttemptsToIssue = TotalLoginAttemptsToIssue; + expConfig1.RecordUnitAttempts = RecordUnitAttempts; + expConfig1.NumberOfIpAddressesControlledByAttacker = MaliciousIP; + //blockConfig1.BlockThresholdPopularPassword = 60; + //blockConfig1.BlockThresholdUnpopularPassword = 900; + //blockConfig1.PenaltyForInvalidAccount = 35; + + + + + + //blockConfig1.PenaltyForReachingEachPopularityThreshold = new List + //{ + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(100*1000d), Penalty = 10*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(10*1000d), Penalty = 20*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(1*1000d), Penalty = 25*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(100d), Penalty = 30*Math.Pow(2,0.3*i)}, + //}; + Simulator simulator = new Simulator(expConfig1, blockConfig1); + Console.WriteLine("PenaltyForInvalidAccount {0}", blockConfig1.PenaltyForInvalidAccount); + + await simulator.Run(blockConfig1, "PenaltyForInvalidAccount"); + } + //4.Vary RewardForCorrectPasswordPerAccount from -10 to -100 + for (int i = 0; i < 10; i++) + { + ExperimentalConfiguration expConfig1 = new ExperimentalConfiguration(); + BlockingAlgorithmOptions blockConfig1 = new BlockingAlgorithmOptions(); + blockConfig1.RewardForCorrectPasswordPerAccount = -10 * i - 10; + //blockConfig1.PenaltyForInvalidAccount = 1 + (int)(i/3); + // blockConfig1.BlockThresholdPopularPassword = 20 + 10 * i; + //blockConfig1.BlockThresholdUnpopularPassword = 4 * blockConfig1.BlockThresholdPopularPassword; + //blockConfig1.BlockThresholdUnpopularPassword = 100 + 200 * i; + expConfig1.TotalLoginAttemptsToIssue = TotalLoginAttemptsToIssue; + expConfig1.RecordUnitAttempts = RecordUnitAttempts; + expConfig1.NumberOfIpAddressesControlledByAttacker = MaliciousIP; + //blockConfig1.BlockThresholdPopularPassword = 60; + //blockConfig1.BlockThresholdUnpopularPassword = 900; + //blockConfig1.PenaltyForInvalidAccount = 35; + + + + + //blockConfig1.PenaltyForReachingEachPopularityThreshold = new List + //{ + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(100*1000d), Penalty = 10*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(10*1000d), Penalty = 20*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(1*1000d), Penalty = 25*Math.Pow(2,0.3*i)}, + // new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(100d), Penalty = 30*Math.Pow(2,0.3*i)}, + //}; + Simulator simulator = new Simulator(expConfig1, blockConfig1); + Console.WriteLine("RewardForCorrectPasswordPerAccount {0}", blockConfig1.RewardForCorrectPasswordPerAccount); + + await simulator.Run(blockConfig1, "RewardForCorrectPasswordPerAccount"); + } + + + + //5.PenaltyForReachingEachPopularityThreshold multiply each by 2 ^{ 0.3 * k} k0-10 + + + + for (int i = 0; i < 11; i++) + { + ExperimentalConfiguration expConfig1 = new ExperimentalConfiguration(); + BlockingAlgorithmOptions blockConfig1 = new BlockingAlgorithmOptions(); + //blockConfig1.RewardForCorrectPasswordPerAccount = -10 * i - 10; + //blockConfig1.PenaltyForInvalidAccount = 1 + (int)(i/3); + // blockConfig1.BlockThresholdPopularPassword = 20 + 10 * i; + //blockConfig1.BlockThresholdUnpopularPassword = 4 * blockConfig1.BlockThresholdPopularPassword; + //blockConfig1.BlockThresholdUnpopularPassword = 100 + 200 * i; + expConfig1.TotalLoginAttemptsToIssue = TotalLoginAttemptsToIssue; + expConfig1.RecordUnitAttempts = RecordUnitAttempts; + expConfig1.NumberOfIpAddressesControlledByAttacker = MaliciousIP; + //blockConfig1.BlockThresholdPopularPassword = 60; + //blockConfig1.BlockThresholdUnpopularPassword = 900; + //blockConfig1.PenaltyForInvalidAccount = 35; + + + + + blockConfig1.PenaltyForReachingEachPopularityThreshold = new List + { + new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(100*1000d), Penalty = 10*Math.Pow(2,0.3*i)}, + new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(10*1000d), Penalty = 20*Math.Pow(2,0.3*i)}, + new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(1*1000d), Penalty = 25*Math.Pow(2,0.3*i)}, + new PenaltyForReachingAPopularityThreshold { PopularityThreshold = 1d/(100d), Penalty = 30*Math.Pow(2,0.3*i)}, + }; + Simulator simulator = new Simulator(expConfig1, blockConfig1); + Console.WriteLine("PenaltyForReachingEachPopularityThreshold{0}", blockConfig1.PenaltyForReachingEachPopularityThreshold); + + await simulator.Run(blockConfig1, "PenaltyForReachingEachPopularityThreshold"); + } + + + + + + + + + //ExperimentalConfiguration expConfig = new ExperimentalConfiguration(); //BlockingAlgorithmOptions blockConfig = new BlockingAlgorithmOptions(); //Simulator simulator = new Simulator(expConfig, blockConfig); //await simulator.Run(); } + + + + + } -} + +} \ No newline at end of file diff --git a/Simulator/SimulatedAccount.cs b/Simulator/SimulatedAccount.cs index cfe9c8e..95f35da 100644 --- a/Simulator/SimulatedAccount.cs +++ b/Simulator/SimulatedAccount.cs @@ -10,5 +10,6 @@ public class SimulatedAccount // public IPAddress PrimaryIp; public ConcurrentBag Cookies = new ConcurrentBag(); public ConcurrentBag ClientAddresses = new ConcurrentBag(); + public bool CorrectClient = true; } } diff --git a/Simulator/Simulator.cs b/Simulator/Simulator.cs index 1f664c4..461620a 100644 --- a/Simulator/Simulator.cs +++ b/Simulator/Simulator.cs @@ -90,7 +90,7 @@ public class Stats /// Evaluate the accuracy of our stopguessing service by sending user logins and malicious traffic /// /// - public async Task Run(BlockingAlgorithmOptions options, CancellationToken cancellationToken = default(CancellationToken)) + public async Task Run(BlockingAlgorithmOptions options, string ParameterSweep, CancellationToken cancellationToken = default(CancellationToken)) { //1.Create account from Rockyou //Create 2*accountnumber accounts, first half is benign accounts, and second half is correct accounts owned by attackers @@ -206,24 +206,44 @@ await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.RecordUnitAttempt Console.Error.WriteLine(e.ToString()); //count++; }); - falsePositiveRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TruePositives); - falseNegativeRate = ((double)stats.FalseNegatives) / ((double)stats.FalseNegatives + stats.TrueNegatives); - detectionRate = ((double)stats.TruePositives) / ((double)stats.TruePositives + stats.FalseNegatives); - falseDetectionRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TrueNegatives); - using (StringWriter filename = new StringWriter()) + if (((double)stats.FalsePositives + stats.TruePositives) != 0) { - filename.Write("Detailed_Log_Unpopular{0}.txt", options.BlockThresholdUnpopularPassword); + falsePositiveRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TruePositives); + - //string filename = "Detailed_Log_Unpopular{0}.txt", options.BlockThresholdUnpopularPassword; - using (StreamWriter detailed = File.AppendText(@filename.ToString())) - { - detailed.WriteLine("The false postive rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalsePositives, stats.TruePositives, falsePositiveRate * 100d); - detailed.WriteLine("The false negative rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalseNegatives, stats.TrueNegatives, falseNegativeRate * 100d); - detailed.WriteLine("The detection rate is {0}/({0}+{1}) ({2:F20}%)", stats.TruePositives, stats.FalseNegatives, detectionRate * 100d); - detailed.WriteLine("The false detection rate is {0}/{0}+({1}) ({2:F20}%)", stats.FalsePositives, stats.TrueNegatives, falseDetectionRate * 100d); - detailed.WriteLine("Start to catch attacker after {0} requests and attackers have login {1} times into users accounts successfully.", stats.bootstrapall, stats.bootstrapsuccess); - } } + if (((double)stats.FalseNegatives + stats.TrueNegatives) != 0) + falseNegativeRate = ((double)stats.FalseNegatives) / ((double)stats.FalseNegatives + stats.TrueNegatives); + if (((double)stats.TruePositives + stats.FalseNegatives) != 0) + detectionRate = ((double)stats.TruePositives) / ((double)stats.TruePositives + stats.FalseNegatives); + if (((double)stats.FalsePositives + stats.TrueNegatives) != 0) + falseDetectionRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TrueNegatives); + //using (StringWriter filename = new StringWriter()) + //{ + // filename.Write("Detailed_PenaltyForReachingAPopularityThreshold{0}.txt", options.PenaltyForReachingEachPopularityThreshold[0]); + + // //string filename = "Detailed_Log_Unpopular{0}.txt", options.BlockThresholdUnpopularPassword; + // using (StreamWriter detailed = File.AppendText(@filename.ToString())) + // { + // detailed.WriteLine("The false postive rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalsePositives, stats.TruePositives, falsePositiveRate * 100d); + // detailed.WriteLine("The false negative rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalseNegatives, stats.TrueNegatives, falseNegativeRate * 100d); + // detailed.WriteLine("The detection rate is {0}/({0}+{1}) ({2:F20}%)", stats.TruePositives, stats.FalseNegatives, detectionRate * 100d); + // detailed.WriteLine("The false detection rate is {0}/{0}+({1}) ({2:F20}%)", stats.FalsePositives, stats.TrueNegatives, falseDetectionRate * 100d); + // detailed.WriteLine("Start to catch attacker after {0} requests and attackers have login {1} times into users accounts successfully.", stats.bootstrapall, stats.bootstrapsuccess); + // } + //} + //using (StringWriter filename = new StringWriter()) + //{ + // // filename.Write("Result.csv", options.BlockThresholdUnpopularPassword); + + // //string filename = "Detailed_Log_Unpopular{0}.txt", options.BlockThresholdUnpopularPassword; + // using (StreamWriter CSV = File.AppendText(@"ResultDetail.csv")) + // { + // CSV.WriteLine("{0},{1},{2},{3},{4},{5},{6}, {7}", options.PenaltyForInvalidAccount, falsePositiveRate * 100d, + // falseNegativeRate * 100d, detectionRate * 100d, falseDetectionRate * 100d, stats.bootstrapall, stats.bootstrapsuccess, falsePositiveRate * 10000d); + + // } + //} } @@ -234,15 +254,77 @@ await TaskParalllel.ParallelRepeat(MyExperimentalConfiguration.RecordUnitAttempt Console.WriteLine("Time Elapsed={0}", sw.Elapsed); Console.WriteLine("the new count is {0}", stats.MaliciousCount + stats.BenignCount); - falsePositiveRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TruePositives); - falseNegativeRate = ((double)stats.FalseNegatives) / ((double)stats.FalseNegatives + stats.TrueNegatives); + //falsePositiveRate = ((double)stats.FalsePositives) / ((double)stats.FalsePositives + stats.TruePositives); + //falseNegativeRate = ((double)stats.FalseNegatives) / ((double)stats.FalseNegatives + stats.TrueNegatives); + + //using (System.IO.StreamWriter file = + //new System.IO.StreamWriter(@"result_log.txt")) + //{ + // file.WriteLine("The false postive rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalsePositives, stats.TruePositives, falsePositiveRate * 100d); + // file.WriteLine("The false negative rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalseNegatives, stats.TrueNegatives, falseNegativeRate * 100d); + // file.WriteLine("Time Elapsed={0}", sw.Elapsed); + //} - using (System.IO.StreamWriter file = - new System.IO.StreamWriter(@"result_log.txt")) + using (StringWriter filename = new StringWriter()) { - file.WriteLine("The false postive rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalsePositives, stats.TruePositives, falsePositiveRate * 100d); - file.WriteLine("The false negative rate is {0}/({0}+{1}) ({2:F20}%)", stats.FalseNegatives, stats.TrueNegatives, falseNegativeRate * 100d); - file.WriteLine("Time Elapsed={0}", sw.Elapsed); + // filename.Write("Result.csv", options.BlockThresholdUnpopularPassword); + + if (ParameterSweep == "BlockThresholdUnpopularPassword") + { + string csvname = "Log_BlockThresholdUnpopularPassword.csv"; + using (StreamWriter CSV = File.AppendText(@csvname)) + { + CSV.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7}", options.BlockThresholdUnpopularPassword, falsePositiveRate * 100d, + falseNegativeRate * 100d, detectionRate * 100d, falseDetectionRate * 100d, stats.bootstrapall, stats.bootstrapsuccess, falsePositiveRate * 10000d); + + } + + + } + else if (ParameterSweep == "BlockThresholdPopularPassword") + { + string csvname = "Log_BlockThresholdPopularPassword.csv"; + using (StreamWriter CSV = File.AppendText(@csvname)) + { + CSV.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7}", options.BlockThresholdPopularPassword, falsePositiveRate * 100d, + falseNegativeRate * 100d, detectionRate * 100d, falseDetectionRate * 100d, stats.bootstrapall, stats.bootstrapsuccess, falsePositiveRate * 10000d); + + } + + + } + else if (ParameterSweep == "PenaltyForInvalidAccount") + { + string csvname = "Log_PenaltyForInvalidAccount.csv"; + using (StreamWriter CSV = File.AppendText(@csvname)) + { + CSV.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7}", options.PenaltyForInvalidAccount, falsePositiveRate * 100d, + falseNegativeRate * 100d, detectionRate * 100d, falseDetectionRate * 100d, stats.bootstrapall, stats.bootstrapsuccess, falsePositiveRate * 10000d); + + } + } + else if (ParameterSweep == "RewardForCorrectPasswordPerAccount") + { + string csvname = "RewardForCorrectPasswordPerAccount.csv"; + using (StreamWriter CSV = File.AppendText(@csvname)) + { + CSV.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7}", options.RewardForCorrectPasswordPerAccount, falsePositiveRate * 100d, + falseNegativeRate * 100d, detectionRate * 100d, falseDetectionRate * 100d, stats.bootstrapall, stats.bootstrapsuccess, falsePositiveRate * 10000d); + + } + } + else if (ParameterSweep == "PenaltyForReachingEachPopularityThreshold") + { + string csvname = "PenaltyForReachingEachPopularityThreshold.csv"; + using (StreamWriter CSV = File.AppendText(@csvname)) + { + CSV.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7}", 1, falsePositiveRate * 100d, + falseNegativeRate * 100d, detectionRate * 100d, falseDetectionRate * 100d, stats.bootstrapall, stats.bootstrapsuccess, falsePositiveRate * 10000d); + + } + } + + } } } diff --git a/Simulator/Simulator_LoginAttemptGeneration.cs b/Simulator/Simulator_LoginAttemptGeneration.cs index 2f667d7..f659924 100644 --- a/Simulator/Simulator_LoginAttemptGeneration.cs +++ b/Simulator/Simulator_LoginAttemptGeneration.cs @@ -51,64 +51,133 @@ public partial class Simulator /// public SimulatedLoginAttempt BenignLoginAttempt() { + + //1. Pick a user at random SimulatedAccount account = BenignAccountSelector.GetItemByWeightedRandom(); - - //2. Deal with cookies + //If the user is from Group 5 , 100/101 of chance a client log in on behalf of the user, 1/101 of chance the user log in string cookie; - // Add a new cookie if there are no cookies, or with if we haven't reached the max number of cookies and lose a roll of the dice - if (account.Cookies.Count == 0 || - (account.Cookies.Count < MyExperimentalConfiguration.MaxCookiesPerUserAccount && StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.ChanceOfCoookieReUse)) - { - // We'll use the decimal represenation of a 64-bit unsigned integer as our cookie - cookie = StrongRandomNumberGenerator.Get64Bits().ToString(); - account.Cookies.Add(cookie); - } - else + IPAddress clientIp; + string password = account.Password; + + if (Int32.Parse(account.UniqueId)>90*1000 && StrongRandomNumberGenerator.GetFraction()<=0.99) { - // Use one of the user's existing cookies selected at random - cookie = account.Cookies.ToArray()[(int)StrongRandomNumberGenerator.Get32Bits(account.Cookies.Count)]; - } + //Everytime the client log in, it has 2% of chance to become a miconfigured client - //Console.WriteLine("The user currently has " + account.Cookies.Count + " cookies. Using: " + cookie); + if (StrongRandomNumberGenerator.GetFraction()<=0.02) + { + account.CorrectClient = false; + } + //2. For cookies, the client always use a specific cookie once the cookie is added, here we assume it's always the first cookie + if(account.Cookies.Count==0) + { + // We'll use the decimal represenation of a 64-bit unsigned integer as our cookie + cookie = StrongRandomNumberGenerator.Get64Bits().ToString(); + account.Cookies.Add(cookie); + } + else + { + cookie = account.Cookies.ToArray()[0]; + + } + + //3. Choose an IP address for the login + // 1/3 of times login with the primary IP address, otherwise, choose an IP randomly from the benign IP pool + + if (account.ClientAddresses.Count == 0 || + (account.ClientAddresses.Count < MyExperimentalConfiguration.MaxIpPerUserAccount && StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.ChanceOfIpReUse)) + { + // Use a new IP for the user + account.ClientAddresses.Add(clientIp = GetNewRandomBenignIp()); + } + else + { + // Use one of the user's existing IP Addresses selected at random + clientIp = account.ClientAddresses.ToArray()[(int)StrongRandomNumberGenerator.Get32Bits(account.ClientAddresses.Count)]; + } + + + if (account.CorrectClient == false) + { + //here we want the password to be the same wrong password all the time, there might be a better way to do it + password = "oldpasswordoldpasswordoldpassword"; + } + else + { + password = account.Password; + } - //3. Choose an IP address for the login - // 1/3 of times login with the primary IP address, otherwise, choose an IP randomly from the benign IP pool - IPAddress clientIp; - if (account.ClientAddresses.Count == 0 || - (account.ClientAddresses.Count < MyExperimentalConfiguration.MaxIpPerUserAccount && StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.ChanceOfIpReUse)) - { - // Use a new IP for the user - account.ClientAddresses.Add(clientIp = GetNewRandomBenignIp()); } + + + + + + else { - // Use one of the user's existing IP Addresses selected at random - clientIp = account.ClientAddresses.ToArray()[(int)StrongRandomNumberGenerator.Get32Bits(account.ClientAddresses.Count)]; - } - - string password = account.Password; + //2. Deal with cookies + //string cookie; + // Add a new cookie if there are no cookies, or with if we haven't reached the max number of cookies and lose a roll of the dice + if (account.Cookies.Count == 0 || + (account.Cookies.Count < MyExperimentalConfiguration.MaxCookiesPerUserAccount && StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.ChanceOfCoookieReUse)) + { + // We'll use the decimal represenation of a 64-bit unsigned integer as our cookie + cookie = StrongRandomNumberGenerator.Get64Bits().ToString(); + account.Cookies.Add(cookie); + } + else + { + // Use one of the user's existing cookies selected at random + cookie = account.Cookies.ToArray()[(int)StrongRandomNumberGenerator.Get32Bits(account.Cookies.Count)]; + } - // - // Add benign failures + //Console.WriteLine("The user currently has " + account.Cookies.Count + " cookies. Using: " + cookie); + + //3. Choose an IP address for the login + // 1/3 of times login with the primary IP address, otherwise, choose an IP randomly from the benign IP pool + //IPAddress clientIp; + if (account.ClientAddresses.Count == 0 || + (account.ClientAddresses.Count < MyExperimentalConfiguration.MaxIpPerUserAccount && StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.ChanceOfIpReUse)) + { + // Use a new IP for the user + account.ClientAddresses.Add(clientIp = GetNewRandomBenignIp()); + } + else + { + // Use one of the user's existing IP Addresses selected at random + clientIp = account.ClientAddresses.ToArray()[(int)StrongRandomNumberGenerator.Get32Bits(account.ClientAddresses.Count)]; + } + + //string password = account.Password; + + // + // Add benign failures + + // The benign user may mistype her password causing a typo (Adding a z will meet the edit distance def. of typo) + if (StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.ChanceOfBenignPasswordTypo) + { + password += "z"; + } + // The benign user may mistakenly use a password for another of her accounts, which we draw from same distribution + // we used to generate user account passwords + if (StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.ChanceOfAccidentallyUsingAnotherAccountPassword) + { + password = GetPasswordFromWeightedDistribution(); + } + // The benign user may mistype her account name, and land on someone else's account name + if (StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.ChanceOfBenignAccountNameTypoResultingInAValidUserName) + { //2% username typo + account = GetBenignAccountAtRandomUniform(); + } - // The benign user may mistype her password causing a typo (Adding a z will meet the edit distance def. of typo) - if (StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.ChanceOfBenignPasswordTypo) - { - password += "z"; - } - // The benign user may mistakenly use a password for another of her accounts, which we draw from same distribution - // we used to generate user account passwords - if (StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.ChanceOfAccidentallyUsingAnotherAccountPassword) - { - password = GetPasswordFromWeightedDistribution(); - } - // The benign user may mistype her account name, and land on someone else's account name - if (StrongRandomNumberGenerator.GetFraction() < MyExperimentalConfiguration.ChanceOfBenignAccountNameTypoResultingInAValidUserName) - { //2% username typo - account = GetBenignAccountAtRandomUniform(); } + + + + + return new SimulatedLoginAttempt(account, password, false, false, clientIp, cookie, DateTimeOffset.Now); }