1
2
3
4 """ handles doing cross validation with k-nearest neighbors model
5
6 and evaluation of individual models
7
8 """
9 from __future__ import print_function
10 from rdkit.ML.KNN.KNNClassificationModel import KNNClassificationModel
11 from rdkit.ML.KNN.KNNRegressionModel import KNNRegressionModel
12 from rdkit.ML.KNN import DistFunctions
13 from rdkit.ML.Data import SplitData
14
19
21 """
22 Determines the classification error for the testExamples
23
24 **Arguments**
25
26 - tree: a decision tree (or anything supporting a _ClassifyExample()_ method)
27
28 - testExamples: a list of examples to be used for testing
29
30 - appendExamples: a toggle which is passed along to the tree as it does
31 the classification. The trees can use this to store the examples they
32 classify locally.
33
34 **Returns**
35
36 a 2-tuple consisting of:
37 """
38 nTest = len(testExamples)
39
40 if isinstance(knnMod,KNNClassificationModel):
41 badExamples = []
42 nBad = 0
43 for i in range(nTest):
44 testEx = testExamples[i]
45 trueRes = testEx[-1]
46 res = knnMod.ClassifyExample(testEx, appendExamples)
47 if (trueRes != res) :
48 badExamples.append(testEx)
49 nBad += 1
50 return float(nBad)/nTest, badExamples
51 elif isinstance(knnMod,KNNRegressionModel):
52 devSum=0.0
53 for i in range(nTest):
54 testEx = testExamples[i]
55 trueRes = testEx[-1]
56 res = knnMod.PredictExample(testEx, appendExamples)
57 devSum += abs(trueRes-res)
58 return devSum/nTest,None
59 raise ValueError("Unrecognized Model Type")
60
68 """ Driver function for building a KNN model of a specified type
69
70 **Arguments**
71
72 - examples: the full set of examples
73
74 - numNeigh: number of neighbors for the KNN model (basically k in k-NN)
75
76 - knnModel: the type of KNN model (a classification vs regression model)
77
78 - holdOutFrac: the fraction of the data which should be reserved for the hold-out set
79 (used to calculate error)
80
81 - silent: a toggle used to control how much visual noise this makes as it goes
82
83 - calcTotalError: a toggle used to indicate whether the classification error
84 of the tree should be calculated using the entire data set (when true) or just
85 the training hold out set (when false)
86 """
87
88 nTot = len(examples)
89 if not kwargs.get('replacementSelection',0):
90 testIndices,trainIndices = SplitData.SplitIndices(nTot,holdOutFrac,
91 silent=1,legacy=1,
92 replacement=0)
93 else:
94 testIndices,trainIndices = SplitData.SplitIndices(nTot,holdOutFrac,
95 silent=1,legacy=0,
96 replacement=1)
97 trainExamples = [examples[x] for x in trainIndices]
98 testExamples = [examples[x] for x in testIndices]
99
100
101 nTrain = len(trainExamples)
102
103 if not silent:
104 print("Training with %d examples"%(nTrain))
105
106 knnMod = modelBuilder(numNeigh, attrs, distFunc)
107
108 knnMod.SetTrainingExamples(trainExamples)
109 knnMod.SetTestExamples(testExamples)
110
111 if not calcTotalError:
112 xValError,badExamples = CrossValidate(knnMod, testExamples,appendExamples=1)
113 else:
114 xValError,badExamples = CrossValidate(knnMod, examples,appendExamples=0)
115
116 if not silent :
117 print('Validation error was %%%4.2f'%(100*xValError))
118
119 knnMod._trainIndices = trainIndices
120 return knnMod, xValError
121