RrdExporter


1   /* ============================================================
2    * JRobin : Pure java implementation of RRDTool's functionality
3    * ============================================================
4    *
5    * Project Info:  http://www.jrobin.org
6    * Project Lead:  Sasa Markovic (saxon@jrobin.org)
7    *
8    * Developers:    Sasa Markovic (saxon@jrobin.org)
9    *                Arne Vandamme (cobralord@jrobin.org)
10   *
11   * (C) Copyright 2003, by Sasa Markovic.
12   *
13   * This library is free software; you can redistribute it and/or modify it under the terms
14   * of the GNU Lesser General Public License as published by the Free Software Foundation;
15   * either version 2.1 of the License, or (at your option) any later version.
16   *
17   * This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
18   * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
19   * See the GNU Lesser General Public License for more details.
20   *
21   * You should have received a copy of the GNU Lesser General Public License along with this
22   * library; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330,
23   * Boston, MA 02111-1307, USA.
24   */
25  package org.jrobin.graph;
26  
27  import java.io.IOException  ;
28  import java.util.HashMap  ;
29  import java.util.ArrayList  ;
30  
31  import org.jrobin.core.*;
32  
33  /**
34   * <p>RrdExporter takes care of calculating a reduced dataset based on a RrdExportDef.</p>
35   *
36   * @author Arne Vandamme (cobralord@jrobin.org)
37   */
38  class RrdExporter
39  {
40      private RrdExportDef def;
41      private RrdOpener rrdOpener;
42  
43      protected int numRows, reducedNumRows;                              // Actual number of rows in the data set
44      protected long startTime, endTime;                                  // Exact (requested) start and end time
45      protected long reducedStartTime, reducedEndTime, reducedStep;       // Reduced start and end time, with step for the reduced set
46  
47      protected long[] timestamps;
48      protected Source[] sources;
49      protected HashMap   sourceIndex;
50  
51      RrdExporter( RrdExportDef def )
52      {
53          setRrdOpener( new RrdOpener( false, true ) );
54          setExportDef( def );
55      }
56  
57      RrdExporter( RrdExportDef def, RrdOpener rrdOpener )
58      {
59          setRrdOpener( rrdOpener );
60          setExportDef( def );
61      }
62  
63      void setExportDef( RrdExportDef def )
64      {
65          this.def            = def;
66      }
67  
68      void setRrdOpener( RrdOpener rrdOpener )
69      {
70          this.rrdOpener = rrdOpener;
71      }
72  
73      /**
74       * Fetches and calculates all datasources requested.
75       *
76       * This method is NOT synchronized, but it is important that access to it is controlled
77       * manually.  This should only be an issue if you are using it for package access, since
78       * the public interface of RrdExport only uses the synchronized fetch method.
79       *
80       * @throws org.jrobin.core.RrdException Thrown in case of a JRobin specific error.
81       * @throws java.io.IOException Thrown in case of a I/O related error.
82       */
83      protected void calculateSeries( int maxRows ) throws RrdException, IOException  
84      {
85          FetchSourceList fetchSources;
86          ValueExtractor ve;
87          FetchSource src;
88          String  [] varList;
89  
90          fetchSources            = def.getFetchSources();
91          fetchSources.setRrdOpener( rrdOpener );
92  
93          long finalEndTime       = Long.MAX_VALUE;
94          boolean changingEndTime = false;
95  
96          long startTime          = def.getStartTime();
97          long endTime            = def.getEndTime();
98          changingEndTime         = (endTime == 0);
99          numRows                 = maxRows;
100         reducedNumRows          = maxRows;
101 
102         int numDefs             = def.getNumDefs();
103         int numSdefs            = def.getNumSdefs();
104 
105         Cdef[] cdefList         = def.getCdefs();
106         int numCdefs            = cdefList.length;
107 
108         Pdef[] pdefList         = def.getPdefs();
109         int numPdefs            = pdefList.length;
110 
111         ExportData[] edata      = def.getExportData();
112         int[] edefTs;
113         Source[] edefList;
114         if ( edata.length > 0 )
115         {
116             ArrayList   tsList    = new ArrayList  ( 30 );
117             ArrayList   list      = new ArrayList  ( 30 );
118             for ( int i = 0; i < edata.length; i++ )
119             {
120                 Source[] esrc = edata[i].getSources();
121 
122                 for ( int j = 0; j < esrc.length; j++ )
123                 {
124                     list.add( esrc[j] );
125                     tsList.add( new Integer  (i) );
126                 }
127             }
128             edefTs              = new int[ tsList.size() ];
129             for ( int i = 0; i < edefTs.length; i++ )
130                 edefTs[i]       = ((Integer  ) tsList.get(i)).intValue();
131             edefList            = (Source[]) list.toArray( new Source[] {} );
132         }
133         else
134         {
135             edefTs              = new int[0];
136             edefList            = new Source[0];
137         }
138         int numEdefs            = edefList.length;
139 
140         // Set up the array with all datasources (both Def, Cdef and Pdef)
141         sources                 = new Source[ numDefs + numEdefs + numCdefs + numPdefs ];
142         sourceIndex             = new HashMap  ( numDefs + numEdefs + numCdefs + numPdefs );
143         int tblPos              = 0;
144         int vePos               = 0;
145 
146         ValueExtractor[] veList = new ValueExtractor[ fetchSources.size() ];
147 
148         long requestedStep      = (long) (endTime - startTime) / maxRows;
149         if ( requestedStep <= 0 ) requestedStep = 1;
150 
151         // Shift start and endTime
152         int minReduceFactor     = 1;
153         long minStep            = Integer.MAX_VALUE, maxStep = Integer.MIN_VALUE, vStartTime, vEndTime, fetchEndTime;
154 
155         // -- Open all fetch datasources
156         if ( fetchSources.size() > 0 || numEdefs > 0 )
157         {
158             try
159             {
160                 fetchSources.openAll();
161 
162                 // Calculate the reduce data factor
163                 for ( int i = 0; i < fetchSources.size(); i++ )
164                 {
165                     src                 = fetchSources.get( i );
166 
167                     if ( changingEndTime )
168                     {
169                         endTime     = src.getLastSampleTime( startTime, endTime, def.getResolution() );
170 
171                         if ( endTime < finalEndTime )
172                             finalEndTime = endTime;
173 
174                         requestedStep = (long) (endTime - startTime) / maxRows;
175                         if ( requestedStep <= 0 ) requestedStep = 1;
176                     }
177 
178                     // Calculate the step for data retrieval
179                     long[] steps        = src.getFetchStep( startTime, endTime, def.getResolution() );
180 
181                     int reduceFactor    = (int) Math.ceil( (double) requestedStep / (double) steps[0] );
182                     steps[0]            = steps[0] * reduceFactor;
183 
184                     if ( steps[0] < minStep )
185                     {
186                         minStep         = steps[0];
187                         minReduceFactor = reduceFactor;
188                     }
189                     if ( steps[1] > maxStep )
190                         maxStep         = steps[1];
191                 }
192 
193                 for ( int i = 0; i < edata.length; i++ )
194                 {
195                     long step = edata[i].getStep();
196 
197                     int reduceFactor    = (int) Math.ceil( (double) requestedStep / (double) step );
198                     step                = step * reduceFactor;
199 
200                     if ( step < minStep )
201                     {
202                         minStep         = step;
203                         minReduceFactor = reduceFactor;
204                     }
205                     if ( step > maxStep )
206                         maxStep         = step;
207                 }
208 
209                 vStartTime          = Util.normalize( startTime, minStep );
210                 vStartTime          = ( vStartTime > startTime ? vStartTime - minStep : vStartTime );
211 
212                 if ( !changingEndTime )
213                 {
214                     vEndTime            = Util.normalize( endTime, minStep );
215                     vEndTime            = ( vEndTime < endTime ? vEndTime + minStep : vEndTime );
216                 }
217                 else
218                 {
219                     vEndTime            = Util.normalize( finalEndTime, minStep );
220                     vEndTime            = ( vEndTime < finalEndTime ? vEndTime + minStep : vEndTime );
221                 }
222 
223                 // This is the actual end time for the reduced data set
224                 reducedEndTime          = vEndTime;
225                 reducedStartTime        = vStartTime;
226                 reducedStep             = minStep;
227                 reducedNumRows          = (int) ((reducedEndTime - reducedStartTime) / reducedStep) + 1;
228 
229                 fetchEndTime            = Util.normalize( vEndTime, maxStep );
230                 fetchEndTime            = ( fetchEndTime < vEndTime ? vEndTime + maxStep : fetchEndTime );
231 
232                 // Now move back to the first time greater than or equal to fetchEndTime, normalized on minStep
233                 vEndTime                = Util.normalize( fetchEndTime, minStep );
234                 vEndTime                = ( vEndTime < fetchEndTime ? vEndTime + minStep : vEndTime );
235 
236                 // Calculate numRows in the end table
237                 numRows                 = (int) ((vEndTime - vStartTime) / minStep) + 1;
238 
239                 // Fetch the actual data
240                 for ( int i = 0; i < fetchSources.size(); i++ )
241                 {
242                     src                 = fetchSources.get( i );
243 
244                     // Fetch all required datasources
245                     ve      = src.fetch( vStartTime, vEndTime, def.getResolution(), minReduceFactor );
246                     varList = ve.getNames();
247 
248                     for (int j= 0; j < varList.length; j++) {
249                         sources[tblPos] = new Def( varList[j], numRows, reducedNumRows );
250                         sourceIndex.put( varList[j], new Integer  (tblPos++) );
251                     }
252 
253                     veList[ vePos++ ] = ve;
254                 }
255             }
256             finally
257             {
258                 // Release all datasources again
259                 fetchSources.releaseAll();
260             }
261         }
262         else
263         {
264             // The range should be used exactly as specified
265             minStep                 = requestedStep;
266             vStartTime              = Util.normalize( startTime, minStep );
267             vStartTime              = ( vStartTime > startTime ? vStartTime - minStep : vStartTime );
268 
269             if ( !changingEndTime )
270             {
271                 vEndTime            = Util.normalize( endTime, minStep );
272                 vEndTime            = ( vEndTime < endTime ? vEndTime + minStep : vEndTime );
273             }
274             else
275             {
276                 vEndTime            = Util.normalize( Util.getTime(), minStep );
277                 vEndTime            = ( vEndTime < endTime ? vEndTime + minStep : vEndTime );
278             }
279 
280             reducedEndTime          = vEndTime;
281             reducedStartTime        = vStartTime;
282             reducedStep             = minStep;
283             reducedNumRows          = (int) ((reducedEndTime - reducedStartTime) / reducedStep) + 1;
284             finalEndTime            = endTime;
285 
286             vEndTime                += minStep;
287             numRows                 = reducedNumRows; //(int) ((vEndTime - vStartTime) / minStep) + 1;
288         }
289 
290         // -- Add all Export datasources to the source table
291         for ( int i = 0; i < edefList.length; i++ )
292         {
293             sources[tblPos] = new Def( edefList[i].getName(), numRows, reducedNumRows );
294             sources[tblPos].setFetchedStep( edefList[i].getStep() );
295             sourceIndex.put( edefList[i].getName(), new Integer  (tblPos++) );
296         }
297 
298         // -- Add all Pdefs to the source table
299         for ( int i = 0; i < pdefList.length; i++ )
300         {
301             pdefList[i].prepare( numRows, reducedNumRows );
302             pdefList[i].setFetchedStep( minStep );
303 
304             sources[tblPos] = pdefList[i];
305             sourceIndex.put( pdefList[i].getName(), new Integer  (tblPos++) );
306         }
307 
308         int cdefStart = tblPos;     // First Cdef element, necessary for tree descend calculation
309 
310         // -- Add all Cdefs to the source table
311         // Reparse all RPN datasources to use indices of the correct variables
312         for ( int i = 0; i < cdefList.length; i++ )
313         {
314             cdefList[i].prepare( sourceIndex, numRows, reducedNumRows );
315             cdefList[i].setFetchedStep( minStep );
316 
317             sources[tblPos] = cdefList[i];
318             sourceIndex.put( cdefList[i].getName(), new Integer  (tblPos++) );
319         }
320 
321         // Fill the array for all datasources
322         timestamps              = new long[numRows];
323 
324         // RPN calculator for the Cdefs
325         RpnCalculator rpnCalc   = new RpnCalculator( sources, minStep );
326 
327         int pos = 0;
328         for (int j = 0; j < veList.length; j++)
329             pos = veList[j].prepareSources( sources, pos );
330 
331         // **************************************************************************************** //
332         // If there are Sdefs, we should determine a tree-descend order for calculation.            //
333         // An Sdef is completely dependant on another datasource and can only be calculated         //
334         // after the datasource it depends on has been calculated itself entirely.                  //
335         //  e.g. The Sdef giving the AVG of a Def should be one lower in the calculation tree       //
336         //       than the corresponding Def.  Lower = higher level.                                 //
337         // Since Sdefs can be nested and combined into new Cdefs and possibly resulting in new      //
338         // Sdefs, the worst case calculation could result in every datasource being calculated      //
339         // separately, resulting in more overhead.  However the impact of this should remain fairly //
340         // small in CPU time.                                                                       //
341         // **************************************************************************************** //
342         if ( numSdefs > 0 )
343         {
344             // Initalize level for each def on 0
345             int treeDepth   = 0;
346             int[] treeLevel = new int[ sources.length ];
347 
348             // First level contains all fetched datasources, custom datasources and combined datasources that use other first levels
349             for ( int i = cdefStart; i < sources.length; i++ )
350             {
351                 // Get the level of all defs needed, take the maximum level
352                 int level       = ((Cdef) sources[i]).calculateLevel( treeLevel );
353                 treeDepth       = (level > treeDepth ? level : treeDepth);
354 
355                 treeLevel[i]    = level;
356             }
357 
358             // Run through each level of the tree
359             long t;
360 
361             for ( int l = 0; l <= treeDepth; l++ )
362             {
363                 t   = vStartTime - minStep;
364                 for ( int i = 0; i < numRows; i++ )
365                 {
366                     pos = cdefStart;
367 
368                     // First level of the tree includes fetched datasources and pdefs,
369                     // since these values can never depend on others in the list.
370                     if ( l == 0 )
371                     {
372                         // Calculate new timestamp
373                         pos = 0;
374                         t   += minStep;
375 
376                         // Get all fetched datasources
377                         for (int j = 0; j < veList.length; j++)
378                             pos = veList[j].extract( t, sources, i, pos );
379 
380                         // Get all export datasources
381                         for (int j = pos; j < pos + numEdefs; j++ )
382                             sources[j].set( i, t, edefList[j - pos].get( t, edata[ edefTs[j - pos] ].getTimestamps() ) );
383                         pos += numEdefs;
384 
385                         // Get all custom datasources
386                         for (int j = pos; j < pos + numPdefs; j++)
387                             ((Pdef) sources[j]).set( i, t );
388                         pos += numPdefs;
389 
390                         timestamps[i] = t;
391                     }
392                     else
393                         t = timestamps[i];
394 
395                     // Calculate the cdefs of this level
396                     for ( int j = pos; j < sources.length; j++ )
397                     {
398                         if ( treeLevel[j] == l )
399                         {
400                             // This Cdef/Sdef can be calculated
401                             if ( sources[j] instanceof Sdef )
402                                 ((Sdef) sources[j]).set( sources );
403                             else
404                                 sources[j].set( i, t, rpnCalc.evaluate( (Cdef) sources[j], i, t ) );
405                         }
406                     }
407                 }
408             }
409         }
410         else
411         {
412             // Traditional way of calculating all datasources, slightly faster
413             long t = vStartTime - minStep;
414             for ( int i = 0; i < numRows; i++ )
415             {
416                 t       += minStep;
417                 pos     = 0;
418 
419                 // Get all fetched datasources
420                 for (int j = 0; j < veList.length; j++)
421                     pos = veList[j].extract( t, sources, i, pos );
422 
423                 // Get all export datasources
424                 for (int j = pos; j < pos + numEdefs; j++ )
425                     sources[j].set( i, t, edefList[j - pos].get( t, edata[ edefTs[j - pos] ].getTimestamps() ) );
426                 pos += numEdefs;
427 
428                 // Get all custom datasources
429                 for (int j = pos; j < pos + numPdefs; j++)
430                     ((Pdef) sources[j]).set( i, t );
431                 pos += numPdefs;
432 
433                 // Get all combined datasources
434                 for (int j = pos; j < sources.length; j++)
435                     sources[j].set(i, t, rpnCalc.evaluate( (Cdef) sources[j], i, t ) );
436 
437                 timestamps[i] = t;
438             }
439         }
440 
441         // Clean up the fetched datasources forcibly
442         veList = null;
443 
444         this.startTime  = startTime;
445         this.endTime    = ( changingEndTime ? finalEndTime : endTime );
446     }
447 
448     private Source getSource( String   name ) throws RrdException
449     {
450         if ( !sourceIndex.containsKey(name) )
451             throw new RrdException( "No such datasource: " + name );
452 
453         return sources[ ( (Integer  ) sourceIndex.get(name) ).intValue() ];
454     }
455 
456     /**
457      * Creates an ExportData object corresponding to the reduced dataset
458      * contained in the RrdExporter.  This assumes that the reduced dataset
459      * has been calculated already!
460      *
461      * @return ExportData object created.
462      * @throws RrdException Thrown in case of JRobin specific error.
463      */
464     protected ExportData createExportData() throws RrdException
465     {
466         if ( sources == null)
467             throw new RrdException( "Sources not calculated, no data to return." );
468         
469         // Now create a RrdDataSet object containing the results
470         Source[] sourceSet;
471         String  [][] export   = def.getExportDatasources();
472         HashMap   legends     = new HashMap  ( export.length );
473 
474         if ( def.isStrict() )
475         {
476             sourceSet           = new Def[ export.length ];
477             for ( int i = 0; i < export.length; i++ )
478                 sourceSet[i] = createReducedDef( getSource( export[i][0] ) );
479         }
480         else
481         {
482             sourceSet           = new Def[ sources.length ];
483             for ( int i = 0; i < sources.length; i++ )
484             {
485                 sourceSet[i] = createReducedDef( sources[i] );
486                 legends.put( sourceSet[i].getName(), sourceSet[i].getName() );
487             }
488         }
489 
490         for ( int i = 0; i < export.length; i++ )
491             legends.put( export[i][0], export[i][1] );
492 
493         long[] reducedTs = new long[ reducedNumRows ];
494         System.arraycopy( timestamps, 0, reducedTs, 0, reducedNumRows );
495 
496         return new ExportData( reducedTs, sourceSet, legends );
497     }
498 
499     private Def createReducedDef( Source origSrc )
500     {
501         Def src = new Def( origSrc.getName(), reducedNumRows, reducedNumRows );
502         src.setFetchedStep( reducedStep );
503 
504         for ( int i = 0; i < reducedNumRows; i++ )
505             src.set( i, timestamps[i], origSrc.get(i) );
506 
507         return src;
508     }
509 
510     /**
511      * Provides a convenient synchronized wrapper around calculateSeries and createExportData.
512      */
513     protected synchronized ExportData fetch( int maxRows ) throws RrdException, IOException  
514     {
515         // Calculate the requested reduced data set
516         calculateSeries( maxRows );
517 
518         return createExportData();
519     }
520 
521     public RrdExportDef getExportDef() {
522         return def;
523     }
524 
525     public RrdOpener getRrdOpener() {
526         return rrdOpener;
527     }
528 }
529
A to Z: JavaDoc & Examples Daily Java News & Articles Open Source Projects Open Source Codes Free Computer Books Remove Frame
Popular Tags