KickJava   Java API By Example, From Geeks To Geeks.

Java > Open Source Codes > mondrian > rolap > FastBatchingCellReader


1 /*
2 // $Id: //open/mondrian/src/main/mondrian/rolap/FastBatchingCellReader.java#38 $
3 // This software is subject to the terms of the Common Public License
4 // Agreement, available at the following URL:
5 // http://www.opensource.org/licenses/cpl.html.
6 // Copyright (C) 2004-2007 Julian Hyde and others
7 // All Rights Reserved.
8 // You must accept the terms of that agreement to use this software.
9 */

10 package mondrian.rolap;
11
12 import mondrian.olap.*;
13 import mondrian.rolap.agg.*;
14 import mondrian.rolap.aggmatcher.AggGen;
15 import mondrian.rolap.sql.SqlQuery;
16
17 import org.apache.log4j.Logger;
18 import org.eigenbase.util.property.*;
19 import org.eigenbase.util.property.Property;
20
21 import java.util.*;
22
23 /**
24  * A <code>FastBatchingCellReader</code> doesn't really Read cells: when asked
25  * to look up the values of stored measures, it lies, and records the fact
26  * that the value was asked for. Later, we can look over the values which
27  * are required, fetch them in an efficient way, and re-run the evaluation
28  * with a real evaluator.
29  *
30  * <p>NOTE: When it doesn't know the answer, it lies by returning an error
31  * object. The calling code must be able to deal with that.</p>
32  *
33  * <p>This class tries to minimize the amount of storage needed to record the
34  * fact that a cell was requested.</p>
35  */

36 public class FastBatchingCellReader implements CellReader {
37
38     private static final Logger LOGGER =
39         Logger.getLogger(FastBatchingCellReader.class);
40
41     /**
42      * This static variable controls the generation of aggregate table sql.
43      */

44     private static boolean generateAggregateSql =
45              MondrianProperties.instance().GenerateAggregateSql.get();
46
47     static {
48         // Trigger is used to lookup and change the value of the
49
// variable that controls generating aggregate table sql.
50
// Using a trigger means we don't have to look up the property eveytime.
51
MondrianProperties.instance().GenerateAggregateSql.addTrigger(
52                 new TriggerBase(true) {
53                     public void execute(Property property, String JavaDoc value) {
54                         generateAggregateSql = property.booleanValue();
55                     }
56                 });
57     }
58
59     private final RolapCube cube;
60     private final Map<BatchKey, Batch> batches;
61     private int requestCount;
62
63     final AggregationManager aggMgr = AggregationManager.instance();
64
65     private final RolapAggregationManager.PinSet pinnedSegments =
66         aggMgr.createPinSet();
67
68     /**
69      * Indicates that the reader given incorrect results.
70      */

71     private boolean dirty;
72
73     public FastBatchingCellReader(RolapCube cube) {
74         this.cube = cube;
75         this.batches = new HashMap<BatchKey, Batch>();
76     }
77
78     public Object JavaDoc get(Evaluator evaluator) {
79         final RolapEvaluator rolapEvaluator = (RolapEvaluator) evaluator;
80         Member[] currentMembers = rolapEvaluator.getMembers();
81         CellRequest request =
82                 RolapAggregationManager.makeRequest(currentMembers, false, false);
83         if (request == null || request.isUnsatisfiable()) {
84             return Util.nullValue; // request out of bounds
85
}
86         // Try to retrieve a cell and simultaneously pin the segment which
87
// contains it.
88
Object JavaDoc o = aggMgr.getCellFromCache(request, pinnedSegments);
89
90         if (o == Boolean.TRUE) {
91             // Aggregation is being loaded. (todo: Use better value, or
92
// throw special exception)
93
return RolapUtil.valueNotReadyException;
94         }
95         if (o != null) {
96             return o;
97         }
98         // if there is no such cell, record that we need to fetch it, and
99
// return 'error'
100
recordCellRequest(request);
101         return RolapUtil.valueNotReadyException;
102     }
103
104     public int getMissCount() {
105         return requestCount;
106     }
107
108     void recordCellRequest(CellRequest request) {
109         if (request.isUnsatisfiable()) {
110             return;
111         }
112         ++requestCount;
113
114         BitKey bitkey = request.getConstrainedColumnsBitKey();
115         BatchKey key = new BatchKey(bitkey, request.getMeasure().getStar());
116         Batch batch = batches.get(key);
117         if (batch == null) {
118             batch = new Batch(request);
119             batches.put(key, batch);
120
121             if (LOGGER.isDebugEnabled()) {
122                 StringBuilder JavaDoc buf = new StringBuilder JavaDoc(100);
123                 buf.append("FastBatchingCellReader: bitkey=");
124                 buf.append(request.getConstrainedColumnsBitKey());
125                 buf.append(Util.nl);
126
127                 RolapStar.Column[] columns = request.getConstrainedColumns();
128                 for (RolapStar.Column column : columns) {
129                     buf.append(" ");
130                     buf.append(column);
131                     buf.append(Util.nl);
132                 }
133                 LOGGER.debug(buf.toString());
134             }
135         }
136         batch.add(request);
137     }
138
139     /**
140      * Returns whether this reader has told a lie. This is the case if there
141      * are pending batches to load or if {@link #setDirty(boolean)} has been
142      * called.
143      */

144     boolean isDirty() {
145         return dirty || !batches.isEmpty();
146     }
147
148     boolean loadAggregations()
149     {
150         return loadAggregations(null);
151     }
152
153     /**
154      * Loads pending aggregations, if any.
155      *
156      * @param query the parent query object that initiated this
157      * call
158      *
159      * @return Whether any aggregations were loaded.
160      */

161     boolean loadAggregations(Query query) {
162         long t1 = System.currentTimeMillis();
163
164         requestCount = 0;
165         if (batches.isEmpty() && !dirty) {
166             return false;
167         }
168
169         // Sort the batches into deterministic order.
170
List<Batch> batchList = new ArrayList<Batch>(batches.values());
171         Collections.sort(batchList, BatchComparator.instance);
172
173         // Load batches in turn.
174
for (Batch batch : batchList) {
175             if (query != null) {
176                 query.checkCancelOrTimeout();
177             }
178             (batch).loadAggregation();
179         }
180         batches.clear();
181
182         if (LOGGER.isDebugEnabled()) {
183             long t2 = System.currentTimeMillis();
184             LOGGER.debug("loadAggregation (millis): " + (t2 - t1));
185         }
186
187         dirty = false;
188         return true;
189     }
190
191     /**
192      * Sets the flag indicating that the reader has told a lie.
193      */

194     void setDirty(boolean dirty) {
195         this.dirty = dirty;
196     }
197
198     private static final Logger BATCH_LOGGER = Logger.getLogger(Batch.class);
199
200     class Batch {
201         // these are the CellRequest's constrained columns
202
final RolapStar.Column[] columns;
203         // this is the CellRequest's constrained column BitKey
204
final BitKey constrainedColumnsBitKey;
205         final List<RolapStar.Measure> measuresList = new ArrayList<RolapStar.Measure>();
206         final Set<StarColumnPredicate>[] valueSets;
207
208         public Batch(CellRequest request) {
209             columns = request.getConstrainedColumns();
210             constrainedColumnsBitKey = request.getConstrainedColumnsBitKey();
211             valueSets = new HashSet[columns.length];
212             for (int i = 0; i < valueSets.length; i++) {
213                 valueSets[i] = new HashSet<StarColumnPredicate>();
214             }
215         }
216
217         public void add(CellRequest request) {
218             List<StarColumnPredicate> values = request.getValueList();
219             for (int j = 0; j < columns.length; j++) {
220                 valueSets[j].add(values.get(j));
221             }
222             RolapStar.Measure measure = request.getMeasure();
223             if (!measuresList.contains(measure)) {
224                 assert (measuresList.size() == 0) ||
225                         (measure.getStar() ==
226                         (measuresList.get(0)).getStar()):
227                         "Measure must belong to same star as other measures";
228                 measuresList.add(measure);
229             }
230         }
231
232         /**
233          * This can only be called after the add method has been called.
234          *
235          * @return the RolapStar associated with the Batch's first Measure.
236          */

237         private RolapStar getStar() {
238             RolapStar.Measure measure = measuresList.get(0);
239             return measure.getStar();
240         }
241
242         void loadAggregation() {
243             if (generateAggregateSql) {
244                 RolapCube cube = FastBatchingCellReader.this.cube;
245                 if (cube == null || cube.isVirtual()) {
246                     StringBuilder JavaDoc buf = new StringBuilder JavaDoc(64);
247                     buf.append("AggGen: Sorry, can not create SQL for virtual Cube \"");
248                     buf.append(FastBatchingCellReader.this.cube.getName());
249                     buf.append("\", operation not currently supported");
250                     BATCH_LOGGER.error(buf.toString());
251
252                 } else {
253                     AggGen aggGen = new AggGen(
254                             FastBatchingCellReader.this.cube.getStar(), columns);
255                     if (aggGen.isReady()) {
256                         // PRINT TO STDOUT - DO NOT USE BATCH_LOGGER
257
System.out.println("createLost:" +
258                             Util.nl + aggGen.createLost());
259                         System.out.println("insertIntoLost:" +
260                             Util.nl + aggGen.insertIntoLost());
261                         System.out.println("createCollapsed:" +
262                             Util.nl + aggGen.createCollapsed());
263                         System.out.println("insertIntoCollapsed:" +
264                             Util.nl + aggGen.insertIntoCollapsed());
265                     } else {
266                         BATCH_LOGGER.error("AggGen failed");
267                     }
268                 }
269             }
270
271             long t1 = System.currentTimeMillis();
272
273             AggregationManager aggmgr = AggregationManager.instance();
274             StarColumnPredicate[] predicates =
275                     new StarColumnPredicate[columns.length];
276             for (int j = 0; j < columns.length; j++) {
277                 Set<StarColumnPredicate> valueSet = valueSets[j];
278
279                 StarColumnPredicate predicate;
280                 if (valueSet == null) {
281                     predicate = LiteralStarPredicate.FALSE;
282                 } else {
283                     ValueColumnPredicate[] values =
284                         valueSet.toArray(
285                             new ValueColumnPredicate[valueSet.size()]);
286                     // Sort array to achieve determinism in generated SQL.
287
Arrays.sort(
288                         values,
289                         ValueColumnConstraintComparator.instance);
290
291                     predicate =
292                         new ListColumnPredicate(
293                             columns[j],
294                             Arrays.asList((StarColumnPredicate[]) values));
295                 }
296
297                 predicates[j] = predicate;
298             }
299             // TODO: optimize key sets; drop a constraint if more than x% of
300
// the members are requested; whether we should get just the cells
301
// requested or expand to a n-cube
302

303             // If the database cannot execute "count(distinct ...)", split the
304
// distinct aggregations out.
305
SqlQuery.Dialect dialect = getStar().getSqlQueryDialect();
306             int distinctMeasureCount = getDistinctMeasureCount(measuresList);
307             boolean tooManyDistinctMeasures =
308                 distinctMeasureCount > 0 &&
309                     !dialect.allowsCountDistinct() ||
310                     distinctMeasureCount > 1 &&
311                         !dialect.allowsMultipleCountDistinct();
312             if (tooManyDistinctMeasures) {
313                 while (true) {
314                     // Scan for a measure based upon a distinct aggregation.
315
RolapStar.Measure distinctMeasure =
316                         getFirstDistinctMeasure(measuresList);
317                     if (distinctMeasure == null) {
318                         break;
319                     }
320                     final String JavaDoc expr =
321                         distinctMeasure.getExpression().getGenericExpression();
322                     final List<RolapStar.Measure> distinctMeasuresList =
323                         new ArrayList<RolapStar.Measure>();
324                     for (int i = 0; i < measuresList.size();) {
325                         RolapStar.Measure measure =
326                             measuresList.get(i);
327                         if (measure.getAggregator().isDistinct() &&
328                             measure.getExpression().getGenericExpression().
329                                 equals(expr))
330                         {
331                             measuresList.remove(i);
332                             distinctMeasuresList.add(distinctMeasure);
333                         } else {
334                             i++;
335                         }
336                     }
337                     RolapStar.Measure[] measures =
338                         distinctMeasuresList.toArray(
339                             new RolapStar.Measure[distinctMeasuresList.size()]);
340                     aggmgr.loadAggregation(
341                         measures, columns,
342                         constrainedColumnsBitKey,
343                         predicates,
344                         pinnedSegments);
345                 }
346             }
347
348             final int measureCount = measuresList.size();
349             if (measureCount > 0) {
350                 RolapStar.Measure[] measures =
351                     measuresList.toArray(new RolapStar.Measure[measureCount]);
352                 aggmgr.loadAggregation(
353                     measures, columns,
354                     constrainedColumnsBitKey,
355                     predicates, pinnedSegments);
356             }
357
358             if (BATCH_LOGGER.isDebugEnabled()) {
359                 long t2 = System.currentTimeMillis();
360                 BATCH_LOGGER.debug("Batch.loadAggregation (millis) " + (t2 - t1));
361             }
362         }
363
364         /**
365          * Returns the first measure based upon a distinct aggregation, or null
366          * if there is none.
367          */

368         RolapStar.Measure getFirstDistinctMeasure(
369             List<RolapStar.Measure> measuresList)
370         {
371             for (RolapStar.Measure measure : measuresList) {
372                 if (measure.getAggregator().isDistinct()) {
373                     return measure;
374                 }
375             }
376             return null;
377         }
378
379         /**
380          * Returns the number of the measures based upon a distinct aggregation.
381          */

382         int getDistinctMeasureCount(List<RolapStar.Measure> measuresList) {
383             int count = 0;
384             for (RolapStar.Measure measure : measuresList) {
385                 if (measure.getAggregator().isDistinct()) {
386                     ++count;
387                 }
388             }
389             return count;
390         }
391     }
392
393     /**
394      * This is needed because for a Virtual Cube: two CellRequests
395      * could have the same BitKey but have different underlying
396      * base cubes. Without this, one get the result in the
397      * SegmentArrayQuerySpec addMeasure Util.assertTrue being
398      * triggered (which is what happened).
399      */

400     class BatchKey {
401         BitKey key;
402         RolapStar star;
403         BatchKey(BitKey key, RolapStar star) {
404             this.key = key;
405             this.star = star;
406         }
407         public int hashCode() {
408             return key.hashCode() ^ star.hashCode();
409         }
410         public boolean equals(Object JavaDoc other) {
411             if (other instanceof BatchKey) {
412                 BatchKey bkey = (BatchKey) other;
413                 return key.equals(bkey.key) && star.equals(bkey.star);
414             } else {
415                 return false;
416             }
417         }
418         public String JavaDoc toString() {
419             return star.getFactTable().getTableName() + " " + key.toString();
420         }
421     }
422
423     private static class BatchComparator implements Comparator<Batch> {
424         static final BatchComparator instance = new BatchComparator();
425
426         private BatchComparator() {}
427
428         public int compare(
429             Batch o1, Batch o2) {
430             if (o1.columns.length != o2.columns.length) {
431                 return o1.columns.length - o2.columns.length;
432             }
433             for (int i = 0; i < o1.columns.length; i++) {
434                 int c = o1.columns[i].getName().compareTo(
435                     o2.columns[i].getName());
436                 if (c != 0) {
437                     return c;
438                 }
439             }
440             for (int i = 0; i < o1.columns.length; i++) {
441                 int c = compare(o1.valueSets[i], o2.valueSets[i]);
442                 if (c != 0) {
443                     return c;
444                 }
445             }
446             return 0;
447         }
448
449         <T> int compare(Set<T> set1, Set<T> set2) {
450             if (set1.size() != set2.size()) {
451                 return set1.size() - set2.size();
452             }
453             Iterator<T> iter1 = set1.iterator();
454             Iterator<T> iter2 = set2.iterator();
455             while (iter1.hasNext()) {
456                 T v1 = iter1.next();
457                 T v2 = iter2.next();
458                 int c = Util.compareKey(v1, v2);
459                 if (c != 0) {
460                     return c;
461                 }
462             }
463             return 0;
464         }
465     }
466
467     private static class ValueColumnConstraintComparator
468         implements Comparator<ValueColumnPredicate>
469     {
470         static final ValueColumnConstraintComparator instance =
471             new ValueColumnConstraintComparator();
472
473         private ValueColumnConstraintComparator() {}
474
475         public int compare(
476             ValueColumnPredicate o1,
477             ValueColumnPredicate o2)
478         {
479             Object JavaDoc v1 = o1.getValue();
480             Object JavaDoc v2 = o2.getValue();
481             if (v1.getClass() == v2.getClass() &&
482                 v1 instanceof Comparable JavaDoc) {
483                 return ((Comparable JavaDoc) v1).compareTo(v2);
484             } else {
485                 return v1.toString().compareTo(v2.toString());
486             }
487         }
488     }
489 }
490
491 // End FastBatchingCellReader.java
492
Popular Tags