data_table.py 20.8 KB
Newer Older
1
2
3
4
# -*- coding: utf-8 -*-
"""Data_table class which is the base for the other data types like data_line,
data_complex and data_IQ. The idea is adapted from holoviews (holoviews.org)
and builds on pandas as excellent data processing tool.
5
6
7

Author: Christian Schneider <c.schneider@uibk.ac.at>
Date: 16.03.2018
8
9
10
"""
from .base import data_module_base
import pandas as pd
11
from .plot_style import color_scheme, cc, check_color, color_labels
12
13
import holoviews as hv
import holoviews.operation.datashader as hd
14
15
16
17
18
19
20
21
import numpy as np
import matplotlib.pyplot as plt
import bokeh.plotting as bp
from bokeh.models import HoverTool
import scipy.optimize as sp_opt
import scipy.signal as sp_sig
import scipy.interpolate as sp_intp
from collections import OrderedDict
22
23
hv.extension('bokeh')

24
25
26
27
28
29
30
31
32
33

class data_table(data_module_base):
    """Class for table like data with one or more independent variables and
    one or more dependent variables

    Parameters
    -----------
    data_arrays : list, array, np.array
        Data to save. Format should be
        [array_x, array_y, array_z, ...]
34
35
36
    data_names : list, array, np.array
        Labels for the data. Format should be
        ['Label first axis', 'Label second axis', ...]
37
38
    """

39
    def __init__(self, data_arrays=None, data_names=None):
40
        super().__init__()
41
42
43
44
45
46
        # Create default data and names
        if data_arrays is None:
            data_arrays = [[np.nan], [np.nan]]
        df_names = ['x', 'y', 'z']
        df_names += ['x{}'.format(i) for i in range(3, len(data_arrays))]

47
48
49
50
        # Replace default names by the names given
        if data_names:
            for idx in range(len(data_names)):
                if data_names[idx]:
51
                    df_names[idx] = data_names[idx]
52

53
        # Create dictionary for pandas dataframe
54
55
        tmp_dict = {}
        for idx, d in enumerate(data_arrays):
56
57
            tmp_dict[df_names[idx]] = data_arrays[idx]

58
        # Create dataframe
59
60
        self.df = pd.DataFrame(data=tmp_dict)

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
        # Create easy access variables
        self.name_x = df_names[0]
        self.name_y = df_names[1]

        # Create further variables
        self._fit_executed = False
        self._fit_labeles = None

    def _repr_html_(self):
        """Show pandas dataframe as default representation"""
        return '<h3>data_table</h3>' + self.df.head().to_html()

    def return_x(self):
        return np.array(self.df[self.name_x])[self.idx_min:self.idx_max]

    def return_y(self):
        if '{}_smoothed'.format(self.name_y) in self.df.keys():
            # Return smoothed data
            name = '{}_smoothed'.format(self.name_y)
            return np.array(self.df[name])[self.idx_min:self.idx_max]
        else:
            return np.array(self.df[self.name_y])[self.idx_min:
                                                  self.idx_max]

    @property
    def x(self):
        return self.return_x()

    @x.setter
    def x(self, value):
Christian Schneider's avatar
Christian Schneider committed
91
        self.df[self.name_x][self.idx_min:self.idx_max] = value
92
93
94
95
96
97
98

    @property
    def y(self):
        return self.return_y()

    @y.setter
    def y(self, value):
Christian Schneider's avatar
Christian Schneider committed
99
        self.df[self.name_y][self.idx_min:self.idx_max] = value
100
101
102
103
104
105
106
107
108
109
110

    def rename_x(self, new_name):
        """Rename x variable"""
        self.df = self.df.rename(columns={self.name_x: new_name})
        self.name_x = new_name

    def rename_y(self, new_name):
        """Rename y variable"""
        self.df = self.df.rename(columns={self.name_y: new_name})
        self.name_y = new_name

111
    def import_data(self, data_arrays, data_names=None):
112
113
        """Import data from new array. Naming highly recommended!
        Assumes first given array is the array of the independent variable.
114
115
116
117
        Parameters
        -----------
        data_arrays : list, array, np.array
            List of data arrays. Structure
118
            [[x, x, x, x, ....], [y, y, y,....], [z, z, z, z, ...]
119
120
        name_list : list, array, np.array, None
            List of names for arrays:
121
            ['x', 'y', 'Resistances', ... ]
122
123
        """
        if data_names:
124
            order_names = data_names
125
        else:
126
            order_names = self.df.columns
127

128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
        # Check if column/dependent variable already exists, if not --> create
        for idx, d_name in zip(range(1, len(order_names)+1),
                                  order_names[1:]):
            if d_name not in self.df.keys():
                if (self.df[order_names[0]] == data_arrays[0]).all():
                    # Quick check: If dependent variable is the same as already
                    # given, just add new column
                    self.df[d_name] = data_arrays[idx]
                else:
                    # Fill existing array with np.nan
                    self.df[d_name] = [np.nan for i in
                                       range(len(self.df[order_names[0]]))]

        # Add values if not already added
        for idx, d in zip(range(len(data_arrays[0])), data_arrays[0]):
            # Check if value for independent variable already exists
            df_idxs = self.df.index[self.df[order_names[0]] == d]
            if not df_idxs.empty:
                df_idx = df_idxs[0]  # take first element
147
                for i in range(1, len(data_arrays)):
148
149
150
151
152
153
154
155
                    if np.isfinite(self.df[order_names[i]][df_idx]):
                        # TODO Is running double if added above --> change
                        # Check if entry is np.nan, if not use mean
                        self.df[order_names[i]][df_idx] += data_arrays[i][df_idx]
                        self.df[order_names[i]][df_idx] /= 2
                    else:
                        # Else just take new value
                        self.df[order_names[i][df_idx]] = data_arrays[i][df_idx]
156
            else:
157
158
                self.df = self.df.append({key: value[idx]} for key, value
                                         in zip(order_names, data_arrays))
159
                pass
160
161

    def plot_hv(self, x=None, y=None, height=400, width=800,
162
                title='', color=None, **kwargs):
163
164
165
166
167
168
169
170
171
172
173
174
175
176
        """Plot table with Holoviews

        Parameters
        -----------
        x : None, str, list
            Column name(s) used for x axis
        y : None, str, list
            Column name(s) used for y axis
        height : int
            Height of plot in pixels
        width : int
            Width of plot in pixels
        """
        if x is None:
177
            x_val = self.df.keys()[0]
178
        else:
179
            x_val = x
180
181

        if y is None:
182
            y_val = self.df.keys()[1]
183
        else:
184
            y_val = y
185

186
187
188
189
        if color is None:
            color = color_scheme[0]
        elif color in cc.keys():
            color = cc[color]
190

191
192
193
        # Plot
        s = hv.Scatter(self.df, x_val, y_val, label=title)
        scatter_plots = hd.dynspread(hd.datashade(hv.Curve(s), cmap=[color]))
194
195
196

        return scatter_plots.opts(plot=dict(height=height, width=width,
                                            show_grid=True))
197

198
    # Plotting #################################################################
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
    def plot(self, style='b-o', color=None, xscale=1, yscale=1,
             plot_fit=True, linewidth=1.5, markersize=3,
             fit_linewidth=1, plot_error_bars=True, legend=None,
             legend_pos=0, engine='bokeh', title='', show=True, fig=None,
             fitcolor='r', fit_on_top=False, **kwargs):
        """Plot data and optionally the fit.

        Choose between two plot-engines: 'bokeh' and 'pyplot'

        Parameters
        ----------
        style : str
            Specify plot style in matplotlib language. E.g. 'b-o' for
            blue dots connected with a line.
        color : str
            Color shortcut (eg. 'b') or specific color like '#123456'.
            Type::
            to see available colors.
        xscale : float
            X scaling
        yscale : float
            Y scaling
        plot_fit : bool
            Plot the fit if available
        linewidth : int
            Thickness of lines
        markersize : int
            Size of markers
        fit_linewidth : int
            Thickness of fit line
        plot_error_bars : bool
            Plot error bars
            ToDo: Implement this for bokeh
        legend : list
            Custom legend for plot ['Label 1', 'Label 2']]
        legend_pos : int, str
            Location of legend
        engine : str
            Plot engine. Choose between 'bokeh' and 'pyplot'
        title : str
            Title of the plot
        show : bool
            Directly show plot. Useful if one wants to add labels etc.
            and show plot afterwards.
        fig : object
            Figure to plot into (bokeh/matplotlib figure object)
        fitcolor : str
            Color shortcut (like 'b') or specific color (like #123456) for
            fit
        fit_on_top : bool
            Data over fit or fit over data

        Returns
        --------
        object
            Returns a fit object for bokeh or matplotlib. This is useful, if
            you want to add for example another points, lines, labels, etc
            afterwards.
        """
        x = self.x
        y = self.y

        # Don't show plot if figure is given (normally one does not need this)
        if fig:
            show = False
        # Get fit function
        if plot_fit and self._fit_executed:
            exec(self._fit_function_code)
            possibles = globals().copy()
            possibles.update(locals())
            fitfunc = possibles.get(self._fit_function)
            if not fitfunc:
                raise Exception('Method %s not implemented' %
                                self._fit_function)

274
275
276
277
278
279
280
281
282
        # Check for color specified in style
        if color is None:
            color = check_color(style)

        if color in color_labels:
            c = cc[color]
        else:
            c = color

283
284
        # Bokeh
        if engine in ['bokeh', 'b']:
285
286
287
288
            # Create Figure if no figure is given
            if fig is None:
                tools = ['box_zoom', 'pan', 'wheel_zoom', 'reset',
                         'save', 'hover']
289
290
                fig = bp.figure(plot_width=800, plot_height=400, tools=tools,
                                toolbar_location='above', title=title)
291
292
293
294
295
296
297
298
299
300
301

            # Empty legend if no legend is given
            if legend is None:
                legend = ""

            # Plot Data
            for kw in style:
                if kw == '-':
                    fig.line(x*xscale, y*yscale, line_color=c,
                             line_width=linewidth, legend=legend, **kwargs)
                elif kw == 'o':
Christian Schneider's avatar
Christian Schneider committed
302
                    fig.circle(x, y, fill_color=c, line_color=c,
303
304
                               size=markersize, legend=legend, **kwargs)
            # Plot Fit
305
            if plot_fit and self._fit_executed:
306
307
                fig.line(x*xscale, fitfunc(x, *self._fit_parameters)*yscale,
                         line_color=fitcolor, line_width=fit_linewidth)
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324

            # Format nicer HoverTool
            tooltips = [("x", "$x"), ("y", "$y")]
            fig.select(dict(type=HoverTool)).tooltips = OrderedDict(tooltips)

            # Add labels
            fig.xaxis.axis_label = self.df.columns[0]
            fig.yaxis.axis_label = self.df.columns[1]

            if show:
                bp.show(fig)
            return fig

        # Matplotlib
        elif engine in ['pyplot', 'p']:
            # Data
            plt.title(title)
325
326
327
            plt.plot(x*xscale, y*yscale, style, color=c, linewidth=linewidth,
                     markersize=markersize, **kwargs)

328
329
330
331
332
333
334
335
336
337
338
339
            # Fit
            if not fig:
                try:
                    fig = plt.gcf()
                except:
                    fig = plt.figure()

            if plot_fit and self._fit_executed:
                if not fit_on_top:
                    fit_zorder=0
                else:
                    fit_zorder=99
340
                plt.plot(x*xscale, fitfunc(x, *self._fit_parameters)*yscale,
341
342
343
                         '-', color=fitcolor, linewidth=fit_linewidth,
                         zorder=fit_zorder)

344
345
346
347
348
349
350
            # Labels
            plt.xlabel(self.name_x)
            plt.ylabel(self.name_y)
            # Styling
            plt.grid()
            plt.xlim([x[0]*xscale, x[-1]*xscale])

351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
        elif engine in ['holoviews', 'h']:
            return self.plot_hv(color=color, **kwargs)

        # ToDO: Error Bars
        # if plot_error_bars is True:
        #    plt.errorbar(xsel*xscale,ysel*yscale,self.return_yerr()*yscale,
        #                 self.return_xerr()*xscale,fmt=plot_style)

    def fit(self, fitfunc, fit_p_init, plot=True, plot_init=True,
            plot_params=True, figuresize=(800, 400), labels=None, **kwargs):
        """Fit `fitfunc` to data with initial fit parameters `fit_p_init`

        Note
        -----
            After the fit, the data module will contain the fit, however it
            should be saved again. Don't forget this.

            The fit will be performed on the stored data, not the plotted
            one. If you use xscale or yscale different than the one you
            plot, you should take it into account

        Parameters
        -----------
        fitfunc : func
            Function to fit to. There exist already some functions in the
            DataModule library (DataModule/fit_functions)
        fit_p_init : list
            Initial (guessed) parameters
        plot : bool
            Plot fit after result
        plot_init : bool
            Plot initial guess.
        figuresize : tuple
            Figure size in pixels (width, height). If you forget your glasses,
            you can increase the size.
        labels : list
            Label for fitparameters. E.g. ['offset', 'amplitude', 'fr']
        **kwargs : keywords
            Keywords for fitfunction like maxfev=10000, epsfcn=1e-10,
            factor=0.1, xtol=1e-9, ftol=1e-9...

        At the end of the fit the data module will contain the function
        used to fit and some fit-related functions will be enabled.
        The fit parameters are stored in
        self._fit_parameters, self._fit_par_errors
        and the average error (sigma) in self._fit_data_error.

        Returns
        list, list, float

        """
        xsel, ysel = self.x, self.y
        try:
            fit_p_fit, err = sp_opt.curve_fit(fitfunc, xsel, ysel, fit_p_init,
                                              **kwargs)
        except RuntimeError:
            print('At least one fit did not converge', end=' ', flush=True)
            fit_p_fit = np.array([np.nan for i in fit_p_init])
            err = np.array([[np.nan for i in fit_p_init] for j in fit_p_init])
            raise Exception(RuntimeError)

        if plot:
            # Just use bokeh since we don't want to publish this plot
            fig = bp.figure(title='Fit', plot_width=800, plot_height=400)
            if plot_init:
                # Plot initial parameter guess, fit and data
                fig.line(xsel, fitfunc(xsel, *fit_p_init), color=cc['g'],
                         legend='Init guess')

            fig.circle(xsel, ysel, color=cc['b'], legend='Data')
            fig.line(xsel, fitfunc(xsel, *fit_p_fit), color=cc['r'],
                     legend='Fit')
            bp.show(fig)

        # Save fitfunction as string
        import inspect
        self._fit_executed = True
        code = inspect.getsourcelines(fitfunc)[0]
        self._fit_function = fitfunc.__name__
        self._fit_function_code = ''.join(code)
        self._fit_parameters = fit_p_fit
        self._fit_par_errors = np.sqrt(np.diag(err))
        # Chi squared
        self._fit_data_error = (np.sum((fitfunc(xsel, *fit_p_fit)-ysel)**2) /
                                (len(xsel) - 2))
        self._fit_labels = labels

        if plot_params:
            print(self.fit_pars())

        return fit_p_fit, self._fit_par_errors, self._fit_data_error

    def fit_func(self, x=None):
        """Calculates values of fit function for an x-array/values.

        Parameters
        -----------
        x : None, float, list, np.array
            X values. None means same x data as datamodule.

        ToDo
        -----
        Would like to rename the function to calc_fitfunc(self, x=None)
        """
        if not self._fit_executed:
            print('No fit was executed on this data')
            return

        exec(self._fit_function_code)
        possibles = globals().copy()
        possibles.update(locals())
        fitfunc = possibles.get(self._fit_function)
        if not fitfunc:
            raise Exception('Method %s not implemented' % self._fit_function)

        if x is None:
            return fitfunc(self.x, *self._fit_parameters)
        else:
            return fitfunc(x, *self._fit_parameters)

    def localmin(self, min_threshold=None, npoints=1, mode='clip'):
        """Obtain all the local minimas

        Parameters
        -----------
        min_threshold : float
            Only consider minima below this value
        npoints : int
            How many points on each side to use for the comparison.
        mode : str
            'clip' (def) or 'wrap'. If wrap is used, the data will considered
            periodic-like

        Returns
        --------
        np.array
            x and y values of local minima
        """
        xsel = self.x
        ysel = self.y

        if min_threshold is not None:
            msk = ysel <= min_threshold
            xsel = xsel[msk]
            ysel = ysel[msk]

        min_idx = sp_sig.argrelextrema(ysel, np.less, order=npoints, mode=mode)
        return np.vstack((xsel[min_idx], ysel[min_idx]))

    def localmax(self, max_threshold=None, npoints=1, mode='clip'):
        """Obtain all the local maxima

        Parameters
        -----------
        min_threshold : float
            Only consider maxima above this value
        npoints : int
            How many points on each side to use for the comparison.
        mode : str
            'clip' (def) or 'wrap'. If wrap is used, the data will considered
            periodic-like

        Returns
        --------
        np.array
            x and y values of local maxima
        """
        xsel = self.x
        ysel = self.y

        if max_threshold is not None:
            msk = ysel >= max_threshold
            xsel = xsel[msk]
            ysel = ysel[msk]

        min_idx = sp_sig.argrelextrema(ysel, np.greater, order=npoints,
                                       mode=mode)
        return np.vstack((xsel[min_idx], ysel[min_idx]))

    def smooth(self, nnb=21, polyorder=2):
        """Smooth data using the Savitzky-Golay filter.

        Information
            https://en.wikipedia.org/wiki/Savitzky%E2%80%93Golay_filter

        This has the advantage over moving_average, that the bias of smaller
        local minima/maxima is removed.

        Note
        -----
        Saves the smoothed data as dm.y values. To unsmooth the data run
        `unsmooth()`

        Parameters
        -----------
        nnb : int
            Window length of filter
        polyorder : int
            Polynomial order for the fit
        """
        y_filtered = sp_sig.savgol_filter(self.df[self.name_y], nnb, polyorder)
        self.import_data([self.x, y_filtered],
                         [self.name_x, '{}_smoothed'.format(self.df.keys()[1])])

    def unsmooth(self):
        """Recover unsmoothened y data."""
        try:
            del self.df['{}_smoothed'.format(self.name_y)]
        except KeyError:
            pass

    def interp(self, xnew, kind='cubic'):
        """Interpolates data to new x values.

        Parameters
        -----------
        xnew : list, np.array
            New x array
        kind : str
            Kind of interpolation.
            Choose between 'linear', 'nearest', 'zero', 'slinear', 'quadratic',
            'cubic'

        Returns
        --------
        DataModule
            Returns a new datamodule.

        Example
        --------
            >>> d1 = dm.load_datamodule('foo.dm')
            >>> xnew = np.linspace(d1.x.min(), d1.x.max(), 100)
            >>> d2 = d1.interp(xnew)
        """
        f = sp_intp.interp1d(self.x, self.y, kind=kind)
        return data_xy(xnew, f(xnew))

    def fit_pars(self):
        """Returns the fit parameters as pandas DataFrame."""

        if not self._fit_executed:
            print('No Fit found. Please run a fit first.')
            return

        df = pd.DataFrame([list(self._fit_parameters),
                           list(self._fit_par_errors)])
        df.index = ['Value', 'Error']
        if self._fit_labels is not None:
            df.columns = self._fit_labels

        return df.T

    def y_value(self, x):
        """ Returns y value for given x array or value using linear
        interpolation.

        Parameters
        ----------
        x : list, np.array, float
            x value(s)

        Returns
        --------
        list, np.array, float
            Interpolated y value for given x value(s)
        """
        return np.interp(x, self.x, self.y)